#!/usr/bin/python3 import argparse import copy import functools import logging import subprocess import os import sys import stat import fedora_messaging.api import fedora_messaging.config import fedora_messaging.exceptions from pathlib import Path logging.basicConfig(level=logging.ERROR) logger = logging.getLogger('updates-sync') SOURCE = '/mnt/koji/compose/updates/' FEDORADEST = '/pub/fedora/linux/updates/' FEDORAMODDEST = '/pub/fedora/linux/modular/updates/' FEDORAALTDEST = '/pub/fedora-secondary/updates/' EPELDEST = '/pub/epel/' OSTREESOURCE = '/mnt/koji/compose/ostree/repo/' OSTREEDEST = '/mnt/koji/ostree/repo/' RELEASES = {'f42': {'topic': 'fedora', 'version': '42', 'modules': ['fedora', 'fedora-secondary'], 'repos': {'updates': { 'from': 'f42-updates', 'ostrees': [{'ref': 'fedora/42/%(arch)s/updates/silverblue', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/updates/kinoite', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/updates/sericea', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/updates/onyx', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/updates/cosmic-atomic', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}], 'to': [{'arches': ['x86_64', 'aarch64', 'source'], 'dest': os.path.join(FEDORADEST, '42', 'Everything')}, {'arches': ['ppc64le', 's390x'], 'dest': os.path.join(FEDORAALTDEST, '42', 'Everything')} ]}, 'updates-testing': { 'from': 'f42-updates-testing', 'ostrees': [{'ref': 'fedora/42/%(arch)s/testing/silverblue', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/testing/kinoite', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/testing/sericea', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/testing/onyx', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}, {'ref': 'fedora/42/%(arch)s/testing/cosmic-atomic', 'dest': OSTREEDEST, 'arches': ['x86_64', 'aarch64']}], 'to': [{'arches': ['x86_64', 'aarch64', 'source'], 'dest': os.path.join(FEDORADEST, 'testing', '42', 'Everything')}, {'arches': ['ppc64le', 's390x'], 'dest': os.path.join(FEDORAALTDEST, 'testing', '42', 'Everything')} ]}} }, 'f41': {'topic': 'fedora', 'version': '41', 'modules': ['fedora', 'fedora-secondary'], 'repos': {'updates': { 'from': 'f41-updates', 'ostrees': [{'ref': 'fedora/41/%(arch)s/updates/silverblue', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/updates/kinoite', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/updates/sericea', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/updates/onyx', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}], 'to': [{'arches': ['x86_64', 'aarch64', 'source'], 'dest': os.path.join(FEDORADEST, '41', 'Everything')}, {'arches': ['ppc64le', 's390x'], 'dest': os.path.join(FEDORAALTDEST, '41', 'Everything')} ]}, 'updates-testing': { 'from': 'f41-updates-testing', 'ostrees': [{'ref': 'fedora/41/%(arch)s/testing/silverblue', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/testing/kinoite', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/testing/sericea', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}, {'ref': 'fedora/41/%(arch)s/testing/onyx', 'dest': OSTREEDEST, 'arches': ['x86_64', 'ppc64le', 'aarch64']}], 'to': [{'arches': ['x86_64', 'aarch64', 'source'], 'dest': os.path.join(FEDORADEST, 'testing', '41', 'Everything')}, {'arches': ['ppc64le', 's390x'], 'dest': os.path.join(FEDORAALTDEST, 'testing', '41', 'Everything')} ]}} }, 'epel10.1': {'topic': 'epel', 'version': '10.1', 'modules': ['epel'], 'repos': {'epel': { 'from': 'epel10.1', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, '10.1', 'Everything')}, ]}, 'epel-testing': { 'from': 'epel10.1-testing', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'testing', '10.1', 'Everything')}, ]}} }, 'epel10.0': {'topic': 'epel', 'version': '10.0', 'modules': ['epel'], 'repos': {'epel': { 'from': 'epel10.0', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, '10.0', 'Everything')}, ]}, 'epel-testing': { 'from': 'epel10.0-testing', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'testing', '10.0', 'Everything')}, ]}} }, 'epel9': {'topic': 'epel', 'version': '9', 'modules': ['epel'], 'repos': {'epel-testing': { 'from': 'epel9-testing', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'testing', '9', 'Everything')} ]}, 'epel': { 'from': 'epel9', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, '9', 'Everything')} ]}} }, 'epel9n': {'topic': 'epel', 'version': '9', 'modules': ['epel'], 'repos': {'epel-testing': { 'from': 'epel9-next-testing', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'testing', 'next', '9', 'Everything')} ]}, 'epel': { 'from': 'epel9-next', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'next', '9', 'Everything')} ]}} }, 'epel8': {'topic': 'epel', 'version': '8', 'modules': ['epel'], 'repos': {'epel-testing': { 'from': 'epel8-testing', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, 'testing', '8', 'Everything')} ]}, 'epel': { 'from': 'epel8', 'to': [{'arches': ['x86_64', 'aarch64', 'ppc64le', 's390x', 'source'], 'dest': os.path.join(EPELDEST, '8', 'Everything')} ]}} }, } # Beneath this is code, no config needed here def run_command(cmd): logger.info('Running %s', cmd) return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False) def get_ostree_ref(repo, ref): reffile = os.path.join(repo, 'refs', 'heads', ref) if not os.path.exists(reffile): return '----' with open(reffile, 'r') as f: return f.read().split()[0] def sync_ostree(dst, ref): src_commit = get_ostree_ref(OSTREESOURCE, ref) dst_commit = get_ostree_ref(dst, ref) if src_commit == dst_commit: logger.info('OSTree at %s, ref %s in sync', dst, ref) else: # Set the umask to be more permissive so directories get group write # https://github.com/ostreedev/ostree/pull/1984 oldumask = os.umask(0o0002) try: print('Syncing ostree ref %s: %s -> %s' % (ref, src_commit, dst_commit)) logger.info('Syncing OSTree to %s, ref %s: %s -> %s', dst, ref, src_commit, dst_commit) cmd = ['ostree', 'pull-local', '--verbose', '--repo', dst, OSTREESOURCE, ref] out = run_command(cmd) cmd = ['ostree', 'summary', '--verbose', '--repo', dst, '--update'] run_command(cmd) print('Ostree ref %s now at %s' % (ref, src_commit)) finally: os.umask(oldumask) def update_fullfilelist(modules): if not modules: logger.info('No filelists to update') return cmd = ['/usr/local/bin/update-fullfiletimelist', '-l', '/pub/fedora-secondary/update-fullfiletimelist.lock', '-t', '/pub'] cmd.extend(modules) run_command(cmd) def rsync(from_path, to_path, excludes=[], link_dest=None, delete=False): cmd = ['rsync', '-rlptDvHh', '--stats', '--no-human-readable'] for excl in excludes: cmd += ['--exclude', excl] if link_dest: cmd += ['--link-dest', link_dest] if delete: cmd += ['--delete', '--delete-delay'] cmd += [from_path, to_path] stdout = run_command(cmd) results = {'num_bytes': 0, 'num_deleted': 0} for line in stdout.decode('utf-8').split('\n'): if 'Literal data' in line: results['num_bytes'] = int(line.split()[2]) elif 'deleting ' in line: results['num_deleted'] += 1 return results def collect_stats(stats): to_collect = ['num_bytes', 'num_deleted'] totals = {} for stat in to_collect: totals[stat] = functools.reduce(lambda x, y: x + y, [r[stat] for r in stats]) return totals def to_human(num_bytes): ranges = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'] cur_range = 0 while num_bytes >= 1024 and cur_range+1 <= len(ranges): num_bytes = num_bytes / 1024 cur_range += 1 return '%s %s' % (num_bytes, ranges[cur_range]) def sync_single_repo_arch(release, repo, arch, dest_path): source_path = os.path.join(SOURCE, RELEASES[release]['repos'][repo]['from'], 'compose', 'Everything', arch) maindir = 'tree' if arch == 'source' else 'os' results = [] do_drpms = os.path.exists(os.path.join(source_path, maindir, 'drpms')) results.append(rsync(os.path.join(source_path, maindir, 'Packages'), os.path.join(dest_path))) if do_drpms: results.append(rsync(os.path.join(source_path, maindir, 'drpms'), os.path.join(dest_path))) if arch != 'source': results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'), os.path.join(dest_path, 'debug'))) results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'repodata'), os.path.join(dest_path, 'debug'), delete=True)) results.append(rsync(os.path.join(source_path, 'debug', 'tree', 'Packages'), os.path.join(dest_path, 'debug'), delete=True)) results.append(rsync(os.path.join(source_path, maindir, 'repodata'), os.path.join(dest_path), delete=True)) results.append(rsync(os.path.join(source_path, maindir, 'Packages'), os.path.join(dest_path), delete=True)) if do_drpms: results.append(rsync(os.path.join(source_path, maindir, 'drpms'), os.path.join(dest_path), delete=True)) return collect_stats(results) def sync_single_repo(release, repo): global FEDMSG_INITED results = [] for archdef in RELEASES[release]['repos'][repo]['to']: for arch in archdef['arches']: if 'source' in arch: dest_path = os.path.join(archdef['dest'], arch, 'tree') else: dest_path = os.path.join(archdef['dest'], arch) results.append(sync_single_repo_arch(release, repo, arch, dest_path)) stats = collect_stats(results) fedmsg_msg = {'repo': repo, 'release': RELEASES[release]['version'], 'bytes': to_human(stats['num_bytes']), 'raw_bytes': str(stats['num_bytes']), 'deleted': str(stats['num_deleted'])} try: msg = fedora_messaging.api.Message( topic="bodhi.updates.{}.sync".format(RELEASES[release]['topic']), body=fedmsg_msg ) fedora_messaging.api.publish(msg) except fedora_messaging.exceptions.PublishReturned as e: print("Fedora Messaging broker rejected message %s: %s" % (msg.id, e)) except fedora_messaging.exceptions.ConnectionException as e: print("Error sending message %s: %s" % (msg.id, e)) except Exception as e: print("Error sending fedora-messaging message: %s" % (e)) def determine_last_link(release, repo): source_path = os.path.join(SOURCE, RELEASES[release]['repos'][repo]['from']) target = os.readlink(source_path) logger.info('Release %s, repo %s, target %s', release, repo, target) RELEASES[release]['repos'][repo]['from'] = target return target def sync_single_release(release): needssync = False for repo in RELEASES[release]['repos']: target = determine_last_link(release, repo) # if "to" is not empty then sync repo if RELEASES[release]['repos'][repo]['to']: curstatefile = os.path.join( RELEASES[release]['repos'][repo]['to'][0]['dest'], 'state') curstate = None if os.path.exists(curstatefile): with open(curstatefile, 'r') as f: curstate = f.read().split()[0] # Resync if Bodhi failed out during the sync waiting, which leads # to changed repomd.xml without an updated repo. # (updateinfo is inserted again) # Fix: https://github.com/fedora-infra/bodhi/pull/1986 if curstate and curstate == target: curstatestat = os.stat(curstatefile) repostat = os.stat(os.path.join( target, 'compose', 'Everything', RELEASES[release]['repos'][repo]['to'][0]['arches'][0], 'os', 'repodata', 'repomd.xml')) if curstatestat[stat.ST_MTIME] < repostat[stat.ST_MTIME]: # If the curstate file has an earlier mtime than the repomd # of the first architecture, this repo was re-generated # after the first time it got staged. Resync. logger.error( 'Re-stage detected of %s %s. ' 'State mtime: %s, repo mtime: %s', release, repo, curstatestat[stat.ST_MTIME], repostat[stat.ST_MTIME]) curstate = None if curstate and curstate == target: logger.info('This repo has already been synced') else: print('Syncing %s %s from %s -> %s' % (release, repo, curstate, target)) sync_single_repo(release, repo) with open(curstatefile, 'w') as f: f.write(target) needssync = True print('Synced %s %s to %s' % (release, repo, target)) for ostree in RELEASES[release]['repos'][repo].get('ostrees', []): pairs = [] for arch in ostree.get('arches', ['']): dest = ostree['dest'] % {'arch': arch} ref = ostree['ref'] % {'arch': arch} pairs.append((dest, ref)) for pair in pairs: sync_ostree(*pair) return needssync def get_epel_release_rel_path(release, epel_next=False): """ Get the relative path of the epel-{next-}release build """ if epel_next: for path in Path(RELEASES[release]['repos']['epel']['to'][0]['dest']).rglob('epel-next-release*noarch*'): if 'Packages' in str(path) and 'x86_64' in str(path): abs_path = path pkg_relpath = os.path.relpath(path,EPELDEST) else: for path in sorted(Path(RELEASES[release]['repos']['epel']['to'][0]['dest']).rglob('epel-release*noarch*')): if 'Packages' in str(path) and 'x86_64' in str(path): abs_path = str(path) pkg_relpath = os.path.relpath(path,EPELDEST) return abs_path, pkg_relpath def update_epel_release_latest(releases): """ This function, creates or updates a symbolic links for epel-release, latest and next, packages. Creates or updates a symbolic link pointing to the latest release of the epel-release package and another pointing to the next release of the epel-release package. The symbolic link will be created or updated if: - There isn't a symbolic link for the latest package; - Current symbolic link is pointing to an outdated package; - Current symbolic link is broken; - There is a file that isn't a link with the same name of the symbolic link. If the symbolic link is pointing to the latest release already, this function will do nothing. Parameters: releases (dict): contains similar information of global variable RELEASES """ for release in releases: if 'epel' in release: major = RELEASES[release]['version'].split('.')[0] if 'next' in RELEASES[release]['repos']['epel']['to'][0]['dest']: dest = '/pub/epel/epel-next-release-latest-' + major + '.noarch.rpm' # For next's epel release, use the subpackage rpm from epel repo instead of # epel next repo release = release[:-1] abs_path, pkg_relpath = get_epel_release_rel_path(release, True) else: dest = '/pub/epel/epel-release-latest-' + major + '.noarch.rpm' abs_path, pkg_relpath = get_epel_release_rel_path(release) if os.path.lexists(dest) and os.path.islink(dest): origin_dest = os.path.join(EPELDEST,os.readlink(dest)) if os.path.split(origin_dest)[-1] != os.path.split(abs_path)[-1]: os.remove(dest) os.symlink(pkg_relpath, dest) elif os.path.lexists(dest) and not os.path.islink(dest): os.remove(dest) os.symlink(pkg_relpath, dest) else: os.symlink(pkg_relpath, dest) def main(): parser = argparse.ArgumentParser() parser.add_argument("releases", nargs="*", default=RELEASES) parser.add_argument( "--config", dest="config", help="fedora-messaging configuration file to use. " "This allows overriding the default " "/etc/fedora-messaging/config.toml.", ) args = parser.parse_args() if args.config: fedora_messaging.config.conf.load_config(args.config) to_update = [] for release in args.releases: if sync_single_release(release): to_update.extend(RELEASES[release]['modules']) to_update = list(set(to_update)) logger.info('Filelists to update: %s', to_update) update_fullfilelist(to_update) update_epel_release_latest(args.releases) if __name__ == '__main__': main()