#!/bin/python2
import copy
import fedmsg
import functools
import logging
import subprocess
import os
import sys
import stat


logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger('updates-sync')


SOURCE = '/mnt/koji/compose/updates/'
FEDORADEST = '/pub/fedora/linux/updates/'
FEDORAMODDEST = '/pub/fedora/linux/modular/updates/'
FEDORAALTDEST = '/pub/fedora-secondary/updates/'
EPELDEST = '/pub/epel/'
OSTREESOURCE = '/mnt/koji/compose/ostree/repo/'
OSTREEDEST = '/mnt/koji/ostree/repo/'
RELEASES = {'f30': {'topic': 'fedora',
                    'version': '30',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f30-updates',
                        'ostrees': [{'ref': 'fedora/30/x86_64/updates/silverblue',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'armhfp', 'aarch64', 'source'],
                                'dest': os.path.join(FEDORADEST, '30', 'Everything')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '30', 'Everything')}
                              ]},
                              'updates-testing': {
                        'from': 'f30-updates-testing',
                        'ostrees': [{'ref': 'fedora/30/x86_64/testing/silverblue',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '30', 'Everything')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '30', 'Everything')}
                              ]}}
                   },
            'f30m': {'topic': 'fedora',
                    'version': '30m',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f30-modular-updates',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, '30', 'Modular')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '30', 'Modular')}
                              ]},
                              'updates-testing': {
                        'from': 'f30-modular-updates-testing',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '30', 'Modular')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '30', 'Modular')}
                              ]}}
                   },
            'f29': {'topic': 'fedora',
                    'version': '29',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f29-updates',
                        'ostrees': [{'ref': 'fedora/29/%(arch)s/updates/atomic-host',
                                     'dest': OSTREEDEST,
                                     'arches': ['x86_64', 'ppc64le', 'aarch64']},
                                    {'ref': 'fedora/29/x86_64/updates/silverblue',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'armhfp', 'aarch64', 'source'],
                                'dest': os.path.join(FEDORADEST, '29', 'Everything')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '29', 'Everything')}
                              ]},
                              'updates-testing': {
                        'from': 'f29-updates-testing',
                        'ostrees': [{'ref': 'fedora/29/%(arch)s/testing/atomic-host',
                                     'dest': OSTREEDEST,
                                     'arches': ['x86_64', 'ppc64le', 'aarch64']},
                                    {'ref': 'fedora/29/x86_64/testing/silverblue',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '29', 'Everything')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '29', 'Everything')}
                              ]}}
                   },
            'f29m': {'topic': 'fedora',
                    'version': '29m',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f29-modular-updates',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, '29', 'Modular')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '29', 'Modular')}
                              ]},
                              'updates-testing': {
                        'from': 'f29-modular-updates-testing',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '29', 'Modular')},
                               {'arches': ['i386', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '29', 'Modular')}
                              ]}}
                   },
             'f28': {'topic': 'fedora',
                    'version': '28',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f28-updates',
                        'ostrees': [{'ref': 'fedora/28/%(arch)s/updates/atomic-host',
                                     'dest': OSTREEDEST,
                                     'arches': ['x86_64', 'ppc64le', 'aarch64']},
                                    {'ref': 'fedora/28/x86_64/updates/workstation',
                                     'dest': OSTREEDEST},
                                    # Hack around for the fact that ostree on f25 doesn't know links
                                    {'ref': 'fedora/28/x86_64/workstation',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'armhfp', 'aarch64', 'source'],
                                'dest': os.path.join(FEDORADEST, '28', 'Everything')},
                               {'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '28', 'Everything')}
                              ]},
                              'updates-testing': {
                        'from': 'f28-updates-testing',
                        'ostrees': [{'ref': 'fedora/28/%(arch)s/testing/atomic-host',
                                     'dest': OSTREEDEST,
                                     'arches': ['x86_64', 'ppc64le', 'aarch64']},
                                    {'ref': 'fedora/28/x86_64/testing/workstation',
                                     'dest': OSTREEDEST}],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '28', 'Everything')},
                               {'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '28', 'Everything')}
                              ]}}
                   },
            'f28m': {'topic': 'fedora',
                    'version': '28m',
                    'modules': ['fedora', 'fedora-secondary'],
                    'repos': {'updates': {
                        'from': 'f28-modular-updates',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, '28', 'Modular')},
                               {'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, '28', 'Modular')}
                              ]},
                              'updates-testing': {
                        'from': 'f28-modular-updates-testing',
                        'ostrees': [],
                        'to': [{'arches': ['x86_64', 'aarch64', 'armhfp', 'source'],
                                'dest': os.path.join(FEDORADEST, 'testing', '28', 'Modular')},
                               {'arches': ['i386', 'ppc64', 'ppc64le', 's390x'],
                                'dest': os.path.join(FEDORAALTDEST, 'testing', '28', 'Modular')}
                              ]}}
                   },
            'epel7': {'topic': 'epel',
                      'version': '7',
                      'modules': ['epel'],
                      'repos': {'epel-testing': {
                          'from': 'epel7-testing',
                          'to': [{'arches': ['x86_64', 'aarch64', 'ppc64', 'ppc64le', 'source'],
                                  'dest': os.path.join(EPELDEST, 'testing', '7')}
                                ]},
                                'epel': {
                          'from': 'epel7',
                          'to': [{'arches': ['x86_64', 'aarch64', 'ppc64', 'ppc64le', 'source'],
                                  'dest': os.path.join(EPELDEST, '7')}
                                ]}}
                     },
            'epel6': {'topic': 'epel',
                      'version': '6',
                      'modules': ['epel'],
                      'repos': {'epel': {
                          'from': 'dist-6E-epel',
                          'to': [{'arches': ['x86_64', 'ppc64', 'i386', 'source'],
                                  'dest': os.path.join(EPELDEST, '6')}
                                ]},
                                'epel-testing': {
                          'from': 'dist-6E-epel-testing',
                          'to': [{'arches': ['x86_64', 'ppc64', 'i386', 'source'],
                                  'dest': os.path.join(EPELDEST, 'testing', '6')}
                                ]},
                              }
                     },
           }


# Beneath this is code, no config needed here
FEDMSG_INITED = False

def run_command(cmd):
    logger.info('Running %s', cmd)
    return subprocess.check_output(cmd,
                                   stderr=subprocess.STDOUT,
                                   shell=False)


def get_ostree_ref(repo, ref):
    reffile = os.path.join(repo, 'refs', 'heads', ref)
    if not os.path.exists(reffile):
        return '----'
    with open(reffile, 'r') as f:
        return f.read().split()[0]


def sync_ostree(dst, ref):
    src_commit = get_ostree_ref(OSTREESOURCE, ref)
    dst_commit = get_ostree_ref(dst, ref)
    if src_commit == dst_commit:
        logger.info('OSTree at %s, ref %s in sync', dst, ref)
    else:
        print('Syncing ostree ref %s: %s -> %s'
              % (ref, src_commit, dst_commit))
        logger.info('Syncing OSTree to %s, ref %s: %s -> %s',
                    dst, ref, src_commit, dst_commit)
        cmd = ['ostree', 'pull-local', '--verbose', '--repo',
               dst, OSTREESOURCE, ref]
        out = run_command(cmd)
        cmd = ['ostree', 'summary', '--verbose', '--repo', dst, '--update']
        run_command(cmd)
        print('Ostree ref %s now at %s' % (ref, src_commit))


def update_fullfilelist(modules):
    if not modules:
        logger.info('No filelists to update')
        return
    cmd = ['/usr/local/bin/update-fullfiletimelist', '-l',
           '/pub/fedora-secondary/update-fullfiletimelist.lock', '-t', '/pub']
    cmd.extend(modules)
    run_command(cmd)


def rsync(from_path, to_path, excludes=[], link_dest=None, delete=False):
    cmd = ['rsync', '-rlptDvHh', '--stats', '--no-human-readable']
    for excl in excludes:
        cmd += ['--exclude', excl]
    if link_dest:
        cmd += ['--link-dest', link_dest]
    if delete:
        cmd += ['--delete', '--delete-delay']

    cmd += [from_path, to_path]

    stdout = run_command(cmd)

    results = {'num_bytes': 0,
               'num_deleted': 0}
    for line in stdout.split('\n'):
        if 'Literal data' in line:
            results['num_bytes'] = int(line.split()[2])
        elif 'deleting ' in line:
            results['num_deleted'] += 1

    return results


def collect_stats(stats):
    to_collect = ['num_bytes', 'num_deleted']
    totals = {}
    for stat in to_collect:
        totals[stat] = functools.reduce(lambda x, y: x + y,
                                        [r[stat] for r in stats])
    return totals


def to_human(num_bytes):
    ranges = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']
    cur_range = 0
    while num_bytes >= 1024 and cur_range+1 <= len(ranges):
        num_bytes = num_bytes / 1024
        cur_range += 1
    return '%s %s' % (num_bytes, ranges[cur_range])



def sync_single_repo_arch(release, repo, arch, dest_path):
    source_path = os.path.join(SOURCE,
                               RELEASES[release]['repos'][repo]['from'],
                               'compose', 'Everything', arch)

    maindir = 'tree' if arch == 'source' else 'os'

    results = []
    do_drpms = os.path.exists(os.path.join(source_path, maindir, 'drpms'))

    results.append(rsync(os.path.join(source_path, maindir, 'Packages'),
                         os.path.join(dest_path)))
    if do_drpms:
        results.append(rsync(os.path.join(source_path, maindir, 'drpms'),
                             os.path.join(dest_path)))
    if arch != 'source':
        results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'),
                             os.path.join(dest_path, 'debug')))
        results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'repodata'),
                             os.path.join(dest_path, 'debug'),
                             delete=True))
        results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'),
                             os.path.join(dest_path, 'debug'),
                             delete=True))
    results.append(rsync(os.path.join(source_path, maindir, 'repodata'),
                         os.path.join(dest_path),
                         delete=True))
    results.append(rsync(os.path.join(source_path, maindir, 'Packages'),
                         os.path.join(dest_path),
                         delete=True))
    if do_drpms:
        results.append(rsync(os.path.join(source_path, maindir, 'drpms'),
                             os.path.join(dest_path),
                             delete=True))

    return collect_stats(results)


def sync_single_repo(release, repo):
    global FEDMSG_INITED

    results = []

    for archdef in RELEASES[release]['repos'][repo]['to']:
        for arch in archdef['arches']:
            destarch = 'SRPMS' if arch == 'source' else arch
            dest_path = os.path.join(archdef['dest'], destarch)
            results.append(sync_single_repo_arch(release, repo, arch, dest_path))

    stats = collect_stats(results)
    fedmsg_msg = {'repo': repo,
                  'release': RELEASES[release]['version'],
                  'bytes': to_human(stats['num_bytes']),
                  'raw_bytes': str(stats['num_bytes']),
                  'deleted': str(stats['num_deleted'])}

    if not FEDMSG_INITED:
        fedmsg.init(active=True, name='relay_inbound', cert_prefix='ftpsync')
        FEDMSG_INITED = True

    fedmsg.publish(topic='updates.%s.sync' % RELEASES[release]['topic'],
                   modname='bodhi',
                   msg=fedmsg_msg)


def determine_last_link(release, repo):
    source_path = os.path.join(SOURCE,
                               RELEASES[release]['repos'][repo]['from'])
    target = os.readlink(source_path)
    logger.info('Release %s, repo %s, target %s', release, repo, target)
    RELEASES[release]['repos'][repo]['from'] = target
    return target


def sync_single_release(release):
    needssync = False

    for repo in RELEASES[release]['repos']:
        target = determine_last_link(release, repo)

        # if "to" is not empty then sync repo
        if RELEASES[release]['repos'][repo]['to']:
            curstatefile = os.path.join(
                RELEASES[release]['repos'][repo]['to'][0]['dest'], 'state')
            curstate = None
            if os.path.exists(curstatefile):
                with open(curstatefile, 'r') as f:
                    curstate = f.read().split()[0]

            # Resync if Bodhi failed out during the sync waiting, which leads
            # to changed repomd.xml without an updated repo.
            # (updateinfo is inserted again)
            # Fix: https://github.com/fedora-infra/bodhi/pull/1986
            if curstate and curstate == target:
                curstatestat = os.stat(curstatefile)
                repostat = os.stat(os.path.join(
                    target, 'compose', 'Everything',
                    RELEASES[release]['repos'][repo]['to'][0]['arches'][0],
                    'os', 'repodata', 'repomd.xml'))
                if curstatestat[stat.ST_MTIME] < repostat[stat.ST_MTIME]:
                    # If the curstate file has an earlier mtime than the repomd
                    # of the first architecture, this repo was re-generated
                    # after the first time it got staged. Resync.
                    logger.error(
                        'Re-stage detected of %s %s. '
                        'State mtime: %s, repo mtime: %s',
                        release, repo,
                        curstatestat[stat.ST_MTIME],
                        repostat[stat.ST_MTIME])
                    curstate = None

            if curstate and curstate == target:
                logger.info('This repo has already been synced')
            else:
                print('Syncing %s %s from %s -> %s' % (release,
                                                       repo,
                                                       curstate,
                                                       target))
                sync_single_repo(release, repo)
                with open(curstatefile, 'w') as f:
                    f.write(target)
                needssync = True
                print('Synced %s %s to %s' % (release, repo, target))

        for ostree in RELEASES[release]['repos'][repo].get('ostrees', []):
            pairs = []
            for arch in ostree.get('arches', ['']):
                dest = ostree['dest'] % {'arch': arch}
                ref = ostree['ref'] % {'arch': arch}
                pairs.append((dest, ref))

            for pair in pairs:
                sync_ostree(*pair)

    return needssync


def main():
    to_update = []

    for release in sys.argv[1:] or RELEASES:
        if sync_single_release(release):
            to_update.extend(RELEASES[release]['modules'])

    to_update = list(set(to_update))
    logger.info('Filelists to update: %s', to_update)
    update_fullfilelist(to_update)


if __name__ == '__main__':
    main()
