#!/usr/bin/python3
"""
This script updates the packageList ownership in Koji based on repo ownership
in Pagure.

This is heavily based on "owner-sync-pkgdb.j2" which was introduced in commit
8c2130080c by Luke Macken.
"""

# cronjobs are run on releng01.stg
# Looks like:
# /usr/local/bin/owner-sync-pkgdb f19 
# /usr/local/bin/owner-sync-pkgdb dist-5E-epel 
# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
# /usr/local/bin/owner-sync-pkgdb epel7 

import sys
import os
import argparse
import configparser
from urllib.parse import urljoin
import multiprocessing.pool
from math import ceil
from functools import partial

import requests
import koji

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry


def retry_session():
    session = requests.Session()
    retry = Retry(
        total=5,
        read=5,
        connect=5,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


# Ansible configured global variables
STAGING = {{ 'True' if env == 'staging' else 'False' }}
HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org'
IPA_REALM = '{{ ipa_realm }}'
ENV_SUFFIX = '{{ env_suffix }}'
if STAGING:
    PAGURE_URL = 'https://src.stg.fedoraproject.org/'
    PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
else:
    PAGURE_URL = 'https://src.fedoraproject.org/'
    PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1/'
# In case the above variables end up being filled in by Ansible
if not PAGURE_URL.endswith('/'):
    PAGURE_URL = PAGURE_URL + '/'
if not PDC_URL.endswith('/'):
    PDC_URL = PDC_URL + '/'

PDC_TYPES = {
    'rpms': 'rpm',
    'modules': 'module',
    'flatpaks': 'flatpak',
    'container': 'container',
}


RAWHIDE = '{{ FedoraRawhideNumber }}'
EXTRA_ARCH_LIST = {
    'kernel': ('i686', 'noarch'),
    'kernel-xen-2.6': ('i686', 'noarch'),
    'glibc': ('i686',),
    'openssl': ('i686',),
    'sysprof-kmod': ('i686',),
}
VERIFY = True


def usage():
    print('Usage: owner-sync <tag>\n\t<tag>: tag to synchronize owners for',
          file=sys.stderr)
    sys.exit(1)


def get_options():
    # shamelessly stolen from koji CLI
    if STAGING:
        opts = {
            'server': 'https://koji.stg.fedoraproject.org/kojihub',
            'weburl': 'https://koji.stg.fedoraproject.org/koji',
        }
    else:
        opts = {
            'server': 'https://koji.fedoraproject.org/kojihub',
            'weburl': 'https://koji.fedoraproject.org/koji',
        }
    opts['principal'] = 'bodhi/{0}@{1}'.format(HOSTNAME, IPA_REALM)
    opts['keytab'] = '/etc/krb5.bodhi_{0}.keytab'.format(HOSTNAME)

    for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
        if os.access(configFile, os.F_OK):
            f = open(configFile)
            config = configparser.ConfigParser()
            config.read_file(f)
            f.close()
            if config.has_section('koji'):
                for name, value in config.items('koji'):
                    if name in opts:
                        opts[name] = value
    for entry in list(opts.keys()):
        if entry == 'server' or entry == 'weburl':
            pass
    return opts


def get_namespaces_and_version_from_tag(tag):
    if 'container' in tag:
        namespaces = ['container']
        version = tag.split('-')[0].split('f')[1]
    elif 'docker' in tag:
        namespaces = ['container']
        version = tag.split('-')[0].split('f')[1]
    elif 'modular' in tag:
        namespaces = ['flatpaks', 'modules']
        try:
            version = tag.split('-')[0].split('f')[1]
        except IndexError:
            version = RAWHIDE
    elif 'flatpak' in tag:
        namespaces = ['flatpaks']
        version = tag.split('-')[0].split('f')[1]
    elif tag == 'module-package-list':
        # See https://pagure.io/releng/issue/6663
        # and https://pagure.io/fm-orchestrator/issue/333
        namespaces = ['rpms']
        version = RAWHIDE
    else:
        namespaces = ['rpms']
        if tag.startswith('epel'):
            version = tag.split('epel')[1]
        elif tag.startswith('f'):
            version = tag.split('f')[1]
        elif tag.endswith('epel') and tag.startswith('dist'):
            # This is for older EPEL tags such as dit-6E-epel
            version = tag.split('-')[1][:-1]
        else:
            print('Error: an invalid tag was specified', file=sys.stderr)
            sys.exit(1)

    return namespaces, version


def get_branch_and_arches(tag, version):
    if tag.startswith('epel'):
        # Ex: epel7 => epel7
        branch = tag
        arches = ["primary"]
    elif tag.endswith('epel'):
        # Ex: dist-6E-epel => el6
        branch = 'el%s' % version
        arches = ["primary"]
    elif tag == 'module-package-list':
        branch = 'rawhide'
        arches = ["primary"]
    else:
        # Fedora
        if version == RAWHIDE:
            branch = 'rawhide'
        else:
            branch = tag.split('-')[0]

        if STAGING:
            arches = ["primary"]
        else:
            if version <= "26":
                arches = ["primary", "s390"]
            else:
                # Yay!  Everything in primary.
                arches = ["primary"]

    return branch, arches


def get_pdc_project_name_and_branch(session, namespace, project_name,
                                    verbose=False):
    """
    Gets the branches on a project. This function is used for mapping.
    :param namespace: string of the namespace the project is in
    :param project: string of the project
    :param verbose: prints out verbose information
    :return: a tuple containing the string of the project and a list of
    branches
    """
    project_branches_url = '{0}component-branches/'.format(PDC_URL)
    params = dict(
        global_component=project_name,
        type=PDC_TYPES[namespace],
        active=True,
    )
    if verbose:
        print('- Querying {0} {1}'.format(project_branches_url, params))
    project_branches_rv = session.get(
        project_branches_url, params=params, verify=VERIFY, timeout=60)

    # If the project's branches can't be reported, just return no branches and
    # it will be skipped later on
    if not project_branches_rv.ok:
        return project_name, []

    data = project_branches_rv.json()
    return project_name, [branch['name'] for branch in data['results']]


def get_pagure_project_names_from_page(session, namespace, page,
                                       package=None, verbose=False):
    """
    Gets the names of all the Pagure projects on a page. This function is to be
    used for mapping.
    :param namespace: string of the namespace to query for projects
    :param page: int of the page to query at
    :param verbose: prints out verbose information
    :return: list of project names on the page
    """
    url = urljoin(PAGURE_URL, 'api/0/projects?namespace={0}'.format(namespace))
    url = url + '&page={0}&per_page=100&fork=false&short=true'.format(page)
    if package:
        url = url + "&pattern={0}".format(package)
    if verbose:
        print('- Querying {0}'.format(url))
    response = session.get(url, verify=VERIFY, timeout=120)
    if not bool(response):
        print("Failed to talk to %r %r." % (
            response.request.url, response), file=sys.stderr)
        return set()

    names = set()
    for project in response.json()['projects']:
        names.add(project['name'])

    return names


def get_pagure_project_branches(namespace, package=None, verbose=False):
    """
    Gets all the branches of all the Pagure projects in the desired namespace
    :param namespace: string of the namespace to query for projects
    :param verbose: prints out verbose information
    :return: dictionary in the format of {project_name: [branch_one...]}
    """
    first_page_url_path = ('api/0/projects?namespace={0}&fork=false&short=true'
                           '&page=1&per_page=1'.format(namespace))
    first_page_url = urljoin(PAGURE_URL, first_page_url_path)
    if package:
        first_page_url = first_page_url + "&pattern={0}".format(package)
    session = retry_session()
    if verbose:
        print('- Querying {0}'.format(first_page_url))
    first_page_rv = session.get(first_page_url, verify=VERIFY, timeout=120)
    if not bool(first_page_rv):
        print("Failed to talk to %r %r." % (
            first_page_rv.request.url, first_page_rv), file=sys.stderr)
        return {}
    total_projects = first_page_rv.json()['total_projects']
    num_pages = int(ceil(total_projects / 100.0))

    pool = multiprocessing.pool.ThreadPool(8)
    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pagure_project_names_from_page = partial(
        get_pagure_project_names_from_page, session, namespace,
        package=package, verbose=verbose)
    project_names_sets = pool.map(partial_get_pagure_project_names_from_page,
                                  list(range(1, num_pages + 1)))

    if project_names_sets:
        # Combine all the project name sets
        project_names = list(set.union(*project_names_sets))
        # Hopefully save some RAM
        del project_names_sets
    else:
        return {}

    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pdc_project_name_and_branch = partial(
        get_pdc_project_name_and_branch, session, namespace,
        verbose=verbose)
    # Get a list of tuples in the form of (project, [branch...]), then convert
    # that to a dictionary
    project_names_to_branches = dict(pool.map(
        partial_get_pdc_project_name_and_branch, project_names))
    pool.close()
    return project_names_to_branches


def get_pagure_project_owner(namespace, package, verbose=False):
    """
    Gets Pagure projects owner in the desired namespace
    :param namespace: string of the namespace to query for projects
    :param project: string of the project name
    :param verbose: prints out verbose information
    :return: the owner name
    """
    pagure_owner_url = urljoin(PAGURE_URL, 'api/0/{0}/{1}'.format(namespace, package))
    session = retry_session()
    if verbose:
        print('- Querying {0}'.format(pagure_owner_url))
    response = session.get(pagure_owner_url, verify=VERIFY, timeout=120)
    if not bool(response):
        print("Failed to talk to %r." % (
            pagure_owner_url), file=sys.stderr)
        return 'releng' # default to releng as the onwer
    owner = response.json()['access_users']['owner'][0]
    return owner


def set_koji_ownership(tag, namespace, packages, arches, verbose=False):
    koji_login_options = get_options()

    koji_options={
        'krb_rdns': False,
        # About ten minutes.  The default is 12 hours.
        'timeout': 60 * 10,
    }
    for arch in arches:

        # Something weird here.  Where do 'arm' and 'ppc' come from?
        if arch in ['arm', 'ppc']:
            arch = 'primary'

        if arch == 'primary':
            session = koji.ClientSession(
                'https://koji{0}.fedoraproject.org/kojihub'.format(ENV_SUFFIX),
                opts=koji_options,
            )
        else:
            session = koji.ClientSession(
                'https://{0}.koji.fedoraproject.org/kojihub'.format(arch),
                opts=koji_options,
            )

        try:
            session.gssapi_login(koji_login_options['principal'], koji_login_options['keytab'])
        except:
            import traceback
            traceback.print_exc()
            print('Failed to login to "{0}" hub'.format(arch), file=sys.stderr)
            continue

        kojitag = session.getTag(tag)
        if kojitag is None:
            print('Error: tag "{0}" does not exist for arch "{1}"'
                  .format(tag, arch), file=sys.stderr)
            continue

        koji_pkgs = {}

        if verbose:
            print('- Getting a list of packages in Koji from the tag "{0}" '
                  'and arch "{1}"'.format(tag, arch))

        for p in session.listPackages(tagID=tag, inherited=True):
            koji_pkgs[p['package_name']] = p

        for pkg in packages:
            owner = get_pagure_project_owner(namespace, pkg, verbose=verbose)
            if (
                namespace == 'flatpaks'
                and not pkg.startswith('flatpak-')
                and not pkg.endswith('-flatpak')
               ):
                koji_pkg = pkg + '-flatpak'
            else:
                koji_pkg = pkg
            # When the user is created in fedora infra, they wont be added to koji
            # until they logged into koji for the first time
            # Set the owner to 'releng' until they logged into koji
            if not session.getUser(owner):
                owner = 'releng'
            if koji_pkg not in koji_pkgs:
                extra_arches = None
                if koji_pkg in EXTRA_ARCH_LIST:
                    extra_arches = " ".join(EXTRA_ARCH_LIST[koji_pkg])
                if verbose:
                    print('- Adding the package "{0}" to the package list for '
                          'the tag "{1}" on arch "{2}" and applicable extra '
                          'arches'.format(koji_pkg, tag, arch))
                session.packageListAdd(
                    tag, koji_pkg, owner=owner, extra_arches=extra_arches)
            elif koji_pkgs[koji_pkg]['owner_name'] != owner:
                if verbose:
                    print('- Setting the owner on package "{0}" for the tag '
                          '"{1}" on arch "{2}"'
                          .format(koji_pkg, tag, arch))
                try:
                    session.packageListSetOwner(tag, koji_pkg, owner, force=True)
                except:
                    # Onwer is not added to koji yet, setting releng as owner
                    print('- Setting releng as owner on package "{0}" for the tag '
                          '"{1}" on arch "{2}"'
                          .format(koji_pkg, tag, arch))
                    session.packageListSetOwner(tag, koji_pkg, 'releng', force=True)



if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('tag', nargs='+',
                        help='tag to update the package list on')
    parser.add_argument('--package', help='Name of an optional single package to sync.')
    parser.add_argument('--verbose', action='store_true')
    args = parser.parse_args()
    verbose = args.verbose
    tags = args.tag
    package = args.package

    # Get all the info about the tags we are interested in
    unique_namespaces = set()
    tag_info = {}
    for tag in tags:
        namespaces, version = get_namespaces_and_version_from_tag(tag)
        branch, arches = get_branch_and_arches(tag, version)
        tag_info[tag] = {
            'namespaces': namespaces,
            'version': version,
            'branch': branch,
            'arches': arches
        }
        unique_namespaces.update(namespaces)

    # Get all the project to branch mappings for every namespace
    namespace_to_projects = {}
    for namespace in unique_namespaces:
        if verbose:
            print('Querying for all the projects with the namespace "{0}"'
                  .format(namespace))
        namespace_to_projects[namespace] = \
            get_pagure_project_branches(namespace, package=package, verbose=verbose)

    for tag, info in list(tag_info.items()):
        if verbose:
            print('Determining which projects have the namespace "{0}" and '
                  'branch "{1}"'.format(namespace, tag))
        namespaces = info['namespaces']
        pkgs = []
        for namespace in namespaces:
            for pkg, branches in list(namespace_to_projects[namespace].items()):
                if info['branch'] in branches or tag == ('f' + RAWHIDE):
                    # The tag and branch names are the same for "old-style" branches
                    pkgs.append(pkg)
                elif namespace in ('modules', 'flatpaks'):
                    # Add modules to f27-modular-updates even if their only branch is '2.4'
                    # Similarly, flatpaks will be built into f29-flatpak-updates-candidate
                    # if they use the f29 runtime, even from rawhide or stream branches.
                    pkgs.append(pkg)

            # This is a special project, not in dist-git, but which needs to be in
            # the package list.
            if namespace == 'rpms':
                pkgs.append('module-build-macros')

        if verbose:
            print('Setting the Koji ownership and package list on packages in '
                  'the tag "{0}" and namespaces "{1}" and for arches "{2}"'
                  .format(tag, namespaces, ', '.join(info['arches'])))
        set_koji_ownership(tag, namespace, pkgs, info['arches'], verbose=verbose)

