# Copyright © 2020 Interplanetary Database Association e.V.,
# Planetmint and IPDB software contributors.
# SPDX-License-Identifier: (Apache-2.0 AND CC-BY-4.0)
# Code is Apache-2.0 and docs are CC-BY-4.0

"""
A little Python script to do some analysis of the NGINX logs.
To get the relevant NGINX logs:
1. Go to the OMS Portal
2. Create a new Log Search
3. Use a search string such as:

Type=ContainerLog Image="planetmint/nginx_3scale:1.3" GET NOT("Go-http-client") NOT(runscope)

(This gets all logs from the NGINX container, only those with the word "GET",
excluding those with the string "Go-http-client" [internal Kubernetes traffic],
excluding those with the string "runscope" [Runscope tests].)

4. In the left sidebar, at the top, use the dropdown menu to select the time range,
e.g. "Data based on last 7 days". Pay attention to the number of results and
the time series chart in the left sidebar. Are there any spikes?
5. Export the search results. A CSV file will be saved on your local machine.
6. $ python3 analyze.py logs.csv

Thanks to https://gist.github.com/hreeder/f1ffe1408d296ce0591d
"""

import sys
import csv
import re
from dateutil.parser import parse


lineformat = re.compile(r'(?P<ipaddress>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) - - '
                        r'\[(?P<dateandtime>\d{2}\/[a-z]{3}\/\d{4}:\d{2}:\d{2}:\d{2} '
                        r'(\+|\-)\d{4})\] ((\"(GET|POST) )(?P<url>.+)(http\/1\.1")) '
                        r'(?P<statuscode>\d{3}) '
                        r'(?P<bytessent>\d+) '
                        r'(["](?P<refferer>(\-)|(.+))["]) '
                        r'(["](?P<useragent>.+)["])',
                        re.IGNORECASE)

filepath = sys.argv[1]

logline_list = []
with open(filepath) as csvfile:
    csvreader = csv.reader(csvfile, delimiter=',')
    for row in csvreader:
        if row and (row[8] != 'LogEntry'):
            # because the first line is just the column headers, such as 'LogEntry'
            logline = row[8]
            print(logline + '\n')
            logline_data = re.search(lineformat, logline)
            if logline_data:
                logline_dict = logline_data.groupdict()
                logline_list.append(logline_dict)
                # so logline_list is a list of dicts
                # print('{}'.format(logline_dict))

# Analysis

total_bytes_sent = 0
tstamp_list = []

for lldict in logline_list:
    total_bytes_sent += int(lldict['bytessent'])
    dt = lldict['dateandtime']
    # https://tinyurl.com/lqjnhot
    dtime = parse(dt[:11] + " " + dt[12:])
    tstamp_list.append(dtime.timestamp())

print('Number of log lines seen: {}'.format(len(logline_list)))

# Time range
trange_sec = max(tstamp_list) - min(tstamp_list)
trange_days = trange_sec / 60.0 / 60.0 / 24.0
print('Time range seen (days): {}'.format(trange_days))

print('Total bytes sent: {}'.format(total_bytes_sent))

print('Average bytes sent per day (out via GET): {}'.
      format(total_bytes_sent / trange_days))
