#!/usr/bin/python

# This file is part of Fedora Project Infrastructure Ansible
# Repository.
#
# Fedora Project Infrastructure Ansible Repository is free software:
# you can redistribute it and/or modify it under the terms of the GNU
# General Public License as published by the Free Software Foundation,
# either version 3 of the License, or (at your option) any later
# version.
#
# Fedora Project Infrastructure Ansible Repository is distributed in
# the hope that it will be useful, but WITHOUT ANY WARRANTY; without
# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License
# along with Fedora Project Infrastructure Ansible Repository.  If
# not, see <http://www.gnu.org/licenses/>.

import sys
import re
import optparse
import os
import string


'''

Mirror list will go through the file given as an argument and parse
out which releases and architectures were looked for, and by how much.

'''

log_line = [
    r"(?P<host>([\d\.]+|[0-9a-fA-F\:]+))\s",
    r"(?P<identity>\S*)\s",
    r"(?P<user>\S*)\s",
    r"\[(?P<time>.*?)\]\s",
    r'"(?P<request>.*?)"\s',
    r"(?P<status>\d+)\s",
    r"(?P<bytes>\S*)\s",
    r'"(?P<referrer>.*?)"\s',
    r'"(?P<user_agent>.*?)"\s*',
]
pattern = re.compile("".join(log_line))


repo_dict = {
    "epel4" : "epel4",
    "epel4.0" : "epel4",
    "epel4.1" : "epel4",
    "epel4.2" : "epel4",
    "epel4.3" : "epel4",
    "epel4.4" : "epel4",
    "epel4.5" : "epel4",
    "epel4.6" : "epel4",
    "epel4.7" : "epel4",
    "epel4.8" : "epel4",
    "epel4.9" : "epel4",
    "epel4.10" : "epel4",
    "epel4.11" : "epel4",
    "epel4.12" : "epel4",
    "epel4.13" : "epel4",
    "epel4.14" : "epel4",
    "epel4.15" : "epel4",
    "epel4.16" : "epel4",
    "epel4.17" : "epel4",
    "epel4.18" : "epel4",
    "epel4.19" : "epel4",
    "epel4.20" : "epel4",
    "epel5" : "epel5",
    "epel5.0" : "epel5",
    "epel5.1" : "epel5",
    "epel5.2" : "epel5",
    "epel5.3" : "epel5",
    "epel5.4" : "epel5",
    "epel5.5" : "epel5",
    "epel5.6" : "epel5",
    "epel5.7" : "epel5",
    "epel5.8" : "epel5",
    "epel5.9" : "epel5",
    "epel5.10" : "epel5",
    "epel5.11" : "epel5",
    "epel5.12" : "epel5",
    "epel5.13" : "epel5",
    "epel5.14" : "epel5",
    "epel5.15" : "epel5",
    "epel5.16" : "epel5",
    "epel5.17" : "epel5",
    "epel5.18" : "epel5",
    "epel5.19" : "epel5",
    "epel5.20" : "epel5",
    "epel6" : "epel6",
    "epel6.0" : "epel6",
    "epel6.1" : "epel6",
    "epel6.2" : "epel6",
    "epel6.3" : "epel6",
    "epel6.4" : "epel6",
    "epel6.5" : "epel6",
    "epel6.6" : "epel6",
    "epel6.7" : "epel6",
    "epel6.8" : "epel6",
    "epel6.9" : "epel6",
    "epel6.10" : "epel6",
    "epel6.11" : "epel6",
    "epel6.12" : "epel6",
    "epel6.13" : "epel6",
    "epel6.14" : "epel6",
    "epel6.15" : "epel6",
    "epel6.16" : "epel6",
    "epel6.17" : "epel6",
    "epel6.18" : "epel6",
    "epel6.19" : "epel6",
    "epel6.20" : "epel6",
    "epel7" : "epel7",
    "epel7.0" : "epel7",
    "epel7.1" : "epel7",
    "epel7.2" : "epel7",
    "epel7.3" : "epel7",
    "epel7.4" : "epel7",
    "epel7.5" : "epel7",
    "epel7.6" : "epel7",
    "epel7.7" : "epel7",
    "epel7.8" : "epel7",
    "epel7.9" : "epel7",
    "epel7.10" : "epel7",
    "epel7.11" : "epel7",
    "epel7.12" : "epel7",
    "epel7.13" : "epel7",
    "epel7.14" : "epel7",
    "epel7.15" : "epel7",
    "epel7.16" : "epel7",
    "epel7.17" : "epel7",
    "epel7.18" : "epel7",
    "epel7.19" : "epel7",
    "epel7.20" : "epel7",
    "epel8" : "epel8",
    "epel8.0" : "epel8",
    "epel8.1" : "epel8",
    "epel8.2" : "epel8",
    "epel8.3" : "epel8",
    "epel8.4" : "epel8",
    "epel8.5" : "epel8",
    "epel8.6" : "epel8",
    "epel8.7" : "epel8",
    "epel8.8" : "epel8",
    "epel8.9" : "epel8",
    "epel8.10" : "epel8",
    "epel8.11" : "epel8",
    "epel8.12" : "epel8",
    "epel8.13" : "epel8",
    "epel8.14" : "epel8",
    "epel8.15" : "epel8",
    "epel8.16" : "epel8",
    "epel8.17" : "epel8",
    "epel8.18" : "epel8",
    "epel8.19" : "epel8",
    "epel8.20" : "epel8",
    "epel9" : "epel9",
    "epel9.0" : "epel9",
    "epel9.1" : "epel9",
    "epel9.2" : "epel9",
    "epel9.3" : "epel9",
    "epel9.4" : "epel9",
    "epel9.5" : "epel9",
    "epel9.6" : "epel9",
    "epel9.7" : "epel9",
    "epel9.8" : "epel9",
    "epel9.9" : "epel9",
    "epel9.10" : "epel9",
    "epel9.11" : "epel9",
    "epel9.12" : "epel9",
    "epel9.13" : "epel9",
    "epel9.14" : "epel9",
    "epel9.15" : "epel9",
    "epel9.16" : "epel9",
    "epel9.17" : "epel9",
    "epel9.18" : "epel9",
    "epel9.19" : "epel9",
    "epel9.20" : "epel9",
    "epelmodular8" : "epel8",
    "epelnext8" : "epel_next8",
    "epelnext9" : "epel_next9",
    "playgroundepel8" : "playground_epel8",
    "playgroundepel9" : "playground_epel9",
    "rawhide" : "rawhide",
    "frawhide" : "rawhide",
    "rawhidemodular" :  "rawhide_modular",
    "3" : "f03",
    "4" : "f04",
    "5" : "f05",
    "6" : "f06",
    "7" : "f07",
    "8" : "f08",
    "9" : "f09",
    "10" : "f10",
    "11" : "f11",
    "12" : "f12",
    "13" : "f13",
    "14" : "f14",
    "15" : "f15",
    "16" : "f16",
    "17" : "f17",
    "18" : "f18",
    "19" : "f19",
    "20" : "f20",
    "21" : "f21",
    "22" : "f22",
    "23" : "f23",
    "24" : "f24",
    "25" : "f25",
    "26" : "f26",
    "27" : "f27",
    "28" : "f28",
    "29" : "f29",
    "30" : "f30",
    "31" : "f31",
    "32" : "f32",
    "33" : "f33",
    "34" : "f34",
    "35" : "f35",
    "36" : "f36",
    "37" : "f37",
    "38" : "f38",
    "39" : "f39",
    "6.89" : "f07",
    "6.90" : "f07",
    "6.91" : "f07",
    "6.92" : "f07",
    "6.93" : "f07",
    "7.89" : "f08",
    "7.90" : "f08",
    "7.91" : "f08",
    "7.92" : "f08",
    "7.93" : "f08",
    "8.90" : "f09",
    "8.91" : "f09",
    "8.92" : "f09",
    "8.93" : "f09",
    "9.90" : "f10",
    "9.90.1" : "f10",
    "9.91" : "f10",
    "9.92" : "f10",
    "9.93" : "f10",
    "10.89" : "f11",
    "10.90" : "f11",
    "10.91" : "f11",
    "10.92" : "f11",
    "10.93" : "f11",
    "11.89" : "f12",
    "11.90" : "f12",
    "11.91" : "f12",
    "11.92" : "f12",
    "11.93" : "f12",
    "12.89" : "f13",
    "12.90" : "f13",
    "12.91" : "f13",
    "12.92" : "f13",
    "12.93" : "f13",
    "f6.89" : "f07",
    "f6.90" : "f07",
    "f6.91" : "f07",
    "f6.92" : "f07",
    "f6.93" : "f07",
    "f7.89" : "f08",
    "f7.90" : "f08",
    "f7.91" : "f08",
    "f7.92" : "f08",
    "f7.93" : "f08",
    "f8.90" : "f09",
    "f8.91" : "f09",
    "f8.92" : "f09",
    "f8.93" : "f09",
    "f9.90" : "f10",
    "f9.90.1" : "f10",
    "f9.91" : "f10",
    "f9.92" : "f10",
    "f9.93" : "f10",
    "f10.89" : "f11",
    "f10.90" : "f11",
    "f10.91" : "f11",
    "f10.92" : "f11",
    "f10.93" : "f11",
    "f11.89" : "f12",
    "f11.90" : "f12",
    "f11.91" : "f12",
    "f11.92" : "f12",
    "f11.93" : "f12",
    "f12.89" : "f13",
    "f12.90" : "f13",
    "f12.91" : "f13",
    "f12.92" : "f13",
    "f12.93" : "f13",
    'f3'       : 'f03',
    'f4'       : 'f04',
    'f5'       : 'f05',
    'f6'       : 'f06',
    'f7'       : 'f07',
    'f8'       : 'f08',
    'f9'       : 'f09',
    'f03'       : 'f03',
    'f04'       : 'f04',
    'f05'       : 'f05',
    'f06'       : 'f06',
    'f07'       : 'f07',
    'f08'       : 'f08',
    'f09'       : 'f09',
    'f10'       : 'f10',
    'f11'       : 'f11',
    'f12'       : 'f12',
    'f13'       : 'f13',
    'f14'       : 'f14',
    'f15'       : 'f15',
    'f16'       : 'f16',
    'f17'       : 'f17',
    'f18'       : 'f18',
    'f19'       : 'f19',
    'f20'       : 'f20',
    'f21'       : 'f21',
    'f22'       : 'f22',
    'f23'       : 'f23',
    'f24'       : 'f24',
    'f25'       : 'f25',
    'f26'       : 'f26',
    'f27'       : 'f27',
    'f28'       : 'f28',
    'f29'       : 'f29',
    'f30'       : 'f30',
    'f31'       : 'f31',
    'f32'       : 'f32',
    'f33'       : 'f33',
    'f34'       : 'f34',
    'f35'       : 'f35',
    'f36'       : 'f36',
    'f37'       : 'f37',
    'f38'       : 'f38',
    'f39'       : 'f39',
    'fmodular27' : 'modular_f27',
    'fmodular28' : 'modular_f28',
    'fmodular29' : 'modular_f29',
    'fmodular30' : 'modular_f30',
    'fmodular31' : 'modular_f31',
    'fmodular32' : 'modular_f32',
    'fmodular33' : 'modular_f33',
    'fmodular34' : 'modular_f34',
    'fmodular35' : 'modular_f35',
    'fmodular36' : 'modular_f36',
    'fmodular37' : 'modular_f37',
    'fmodular38' : 'modular_f38',
    'fmodular39' : 'modular_f39',
    'modularf27' : 'modular_f27',
    'modularf28' : 'modular_f28',
    'modularf29' : 'modular_f29',
    'modularf30' : 'modular_f30',
    'modularf31' : 'modular_f31',
    'modularf32' : 'modular_f32',
    'modularf33' : 'modular_f33',
    'modularf34' : 'modular_f34',
    'modularf35' : 'modular_f35',
    'modularf36' : 'modular_f36',
    'modularf37' : 'modular_f37',
    'modularf38' : 'modular_f38',
    'modularf39' : 'modular_f39',
    'rhel4'     : 'rhel4',
    'rhel5'     : 'rhel5',
    'rhel6'     : 'rhel6',
    'rhel7'     : 'rhel7',
    'rhel8'     : 'rhel8',
    'rhel9'     : 'rhel9',
    'fciscoopenh26432' : 'cisco_32',
    'fciscoopenh26433' : 'cisco_33',
    'fciscoopenh26434' : 'cisco_34',
    'fciscoopenh26435' : 'cisco_35',
    'fciscoopenh26436' : 'cisco_36',
    'fciscoopenh26437' : 'cisco_37',
    'fciscoopenh26438' : 'cisco_38',
    'fciscoopenh26439' : 'cisco_39',
    'fciscoopenh264rawhide' : 'cisco_rawhide',
}

repo_keys = repo_dict.keys()

def breakoutdate(givendate):
    Apache_Months = {
        'Jan' : '01',
        'Feb' : '02',
        'Mar' : '03',
        'Apr' : '04',
        'May' : '05',
        'Jun' : '06',
        'Jul' : '07',
        'Aug' : '08',
        'Sep' : '09',
        'Oct' : '10',
        'Nov' : '11',
        'Dec' : '12', 
    }

    date_part = givendate.split()
    
    try:
        [day, month, year] = givendate.split(":")[0].split('/')
    except:
        # string out of index because date corrupted?
        [day, month, year ] = ['01', 'Jan', '1970'] # epoch
    ret_str = "%s-%s-%s" % (year, Apache_Months[month], day)
    return ret_str

def breakoutrepo(request):
    try:
        parts = request.split()[1].split("?")[1].split("&")
        repo=""
        arch=""
        for i in parts:
            if 'repo=' in i:
                repo = i.split('=')[1]
            if 'arch=' in i:
                arch = i.split('=')[1]
        return (repo,arch)
    except:
        return ("unknown_repo","unknown_arch")


def figureoutrepo(asked_repo):

    global repo_dict
    global repo_keys

    crap_chars = ['/', '$', '!', '#', '%', '&', "'", '"', "(", ")", "*", "+", ",", "_", ":", ";", "<", ">", "=", "?", "@", "[", "^", "|"]

    spew = asked_repo.lower()
    for char in crap_chars:
        if char in spew:
            spew.split(char)[0]

    f_phrases = ["core", "fedora", "extras", "legacy", "fc"]

    for word in f_phrases:
        if word in spew:
            spew = spew.replace(word, "f")

    repo_phrases = [".newkey", "install", "alpha", "beta", "client", "debug", "devel", "info", "optional", "preview", "released", "source", "testing", "updates"]

    for word in repo_phrases:
        if word in spew:
            spew = spew.replace(word, "")

    if "centosplus" in spew:
        spew = spew.replace("centosplus", "centos")

    if "client" in spew:
        spew = re.sub("client.*", "", spew)
    if "cloud" in spew:
        spew = re.sub("cloud.*", "", spew)
    if "server" in spew:
        spew = re.sub("server.*", "", spew)
    if "workstation" in spew:
        spew = re.sub("workstation.*", "", spew)
    if "-" in spew:
        spew = re.sub("-+", "", spew)

    sanitize = spew.strip()

    if sanitize in repo_dict.keys():
        return repo_dict[sanitize]
    else:
        # sys.stderr.write("asked_repo: %s. Thought it was %s\n" % (asked_repo,spew))
        return "unknown_repo"


def figureoutarch(asked_arch):
    arch_dict = {
        'i386' : 'i386',
        'i486' : 'i386',
        'i586' : 'i386',
        'i686' : 'i386',
        'athlon' : 'i386',
        'pentium' : 'i386',
        'pentium3' : 'i386',
        'pentium4' : 'i386',
        'pentium5' : 'i386',
        'ia32' : 'i386',
        'x86' : 'i386',
        'x86_32' : 'i386',
        'x86_64' : 'x86_64',
        'amd64' : 'x86_64',
        'aarch64' : 'aarch64',
        'alpha' : 'alpha',
        'arm' : 'arm',
        'arm64' : 'aarch64',
        'armhfp' : 'arm',
        'armv3l' : 'arm',
        'armv5tel' : 'arm',
        'armv7hl' : 'arm',
        'ia64' : 'ia64',
        'mips' : 'mips',
        'mips64' : 'mips64',
        'mips64el' : 'mips64',
        'powepc' : 'ppc',
        'ppc' : 'ppc',
        'ppc32' : 'ppc',
        'ppc64' : 'ppc64',
        'ppc64le' : 'ppc64le',
        's390' : 's390',
        's390x' : 's390',
        'sparc' : 'sparc',
        'sparc64' : 'sparc64',
        'tilegx' : 'tilegx',
    }
    spew = asked_arch.split("/")[0]
    spew = spew.split("!")[0]
    spew = spew.split("#")[0]
    spew = spew.split("%")[0]
    spew = spew.split("&")[0]
    spew = spew.split("'")[0]
    spew = spew.split("(")[0]
    spew = spew.split("*")[0]
    spew = spew.split("+")[0]
    spew = spew.split(",")[0]
    spew = spew.split("-")[0]
    spew = spew.split(".")[0]
    spew = spew.split(":")[0]
    spew = spew.split(";")[0]
    spew = spew.split("<")[0]
    spew = spew.split("=")[0]
    spew = spew.split(">")[0]
    spew = spew.split("?")[0]
    spew = spew.split("@")[0]
    spew = spew.split("[")[0]
    spew = spew.split("]")[0]
    spew = spew.split("^")[0]
    spew = spew.split('"')[0]
    spew = spew.split('\\')[0]
    spew = spew.split('|')[0]
    spew = spew.split('$')[0]
    sanitize = spew.lower()

    if sanitize in arch_dict.keys():
        return arch_dict[sanitize]
    else:
        #sys.stderr.write("asked_arch: %s\n" % asked_arch)
        return "unknown_arch"

def parseline(our_line):

    ##
    ## Figure out if line is something we want to work on more
    global pattern

    if (('/metalink' in our_line) or ('/mirrorlist' in our_line)):
        our_blob = pattern.match(our_line)
        if our_blob:
            our_dict = our_blob.groupdict()
            ip       = our_dict['host']
            time     = breakoutdate(our_dict['time']) 
            r,a      = breakoutrepo(our_dict['request'])
            repo     = figureoutrepo(r)
            arch     = figureoutarch(a)
            return "%s %s %s %s" % (time,ip,repo,arch)
        else:
            return ""
    else:
        return ""


def parselog(our_file, out_file):
    our_file = our_file
    yumclients_set = set()
    output_file = out_file
    try:
        data = open(our_file, "r")
    except:
        sys.stderr.write("Unable to open %s\n" % our_file )
        sys.exit(-1)

    for line in data:
        parsed = parseline(line)
        if parsed == "":
            pass
        else:
            yumclients_set.add(parsed)

    data.close()

    our_list = list(yumclients_set)
    our_list.sort()

    try:
        output = open(output_file,"a")
        sys.stdout.write("Outputting data: %s\n" % our_file)
    except:
        sys.stderr.write("Unable to open outputfile: %s\n" % our_file)
        sys.exit(-1)

    for line in our_list:
        output.write(line + os.linesep)
    output.close()
    return


def main():
    parser = optparse.OptionParser(
        description = "A program to parse Fedora mirrorlist apache common log format files.",
        prog = "mirrorlist.py",
        version = "1.0.2",
        usage = "%prog [-o output-filename] logfile1 [logfile2...]"
    )

    parser.add_option("-o", "--output",
                      default = "output.txt",
                      help = "Sets the name of the output file for the run.",
                      dest = "output")


    (options, args) = parser.parse_args()
    if options.output:
        out_file = options.output
    else:
        out_file = "output.txt"

    for our_file in args:
        parselog(our_file,out_file)


if __name__ == '__main__':
    main()
