diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index fe00980bae..56bd12edbc 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -116,7 +116,7 @@ # bodhi2/backend/files/koji_sync_listener.py # This cronjob runs only once a day. The listener script runs reactively. cron: name="owner-sync" minute="15" hour="4" user="root" - job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel9 epel9-next epel8'" + job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel10.0 epel9 epel9-next epel8'" cron_file=update-koji-owner user=apache when: env == "production" diff --git a/roles/bodhi2/backend/templates/koji_sync_listener.toml b/roles/bodhi2/backend/templates/koji_sync_listener.toml index 7ba3c3affc..56d2b477c7 100644 --- a/roles/bodhi2/backend/templates/koji_sync_listener.toml +++ b/roles/bodhi2/backend/templates/koji_sync_listener.toml @@ -45,6 +45,7 @@ taglist = [ "f39", "f39-container", "f39-flatpak", + "epel10.0", "epel9", "epel9-next", "epel8", diff --git a/roles/bodhi2/backend/templates/owner-sync-pagure.j2 b/roles/bodhi2/backend/templates/owner-sync-pagure.j2 index 68d5c5fb26..4b9f2648fa 100755 --- a/roles/bodhi2/backend/templates/owner-sync-pagure.j2 +++ b/roles/bodhi2/backend/templates/owner-sync-pagure.j2 @@ -108,89 +108,6 @@ def get_options(): return opts -def get_namespaces_and_version_from_tag(tag): - if 'container' in tag: - namespaces = ['container'] - version = tag.split('-')[0].split('f')[1] - elif 'docker' in tag: - namespaces = ['container'] - version = tag.split('-')[0].split('f')[1] - elif 'modular' in tag: - namespaces = ['flatpaks', 'modules'] - try: - version = tag.split('-')[0].split('f')[1] - except IndexError: - version = RAWHIDE - elif 'flatpak' in tag: - namespaces = ['flatpaks'] - version = tag.split('-')[0].split('f')[1] - elif tag == 'module-package-list': - # See https://pagure.io/releng/issue/6663 - # and https://pagure.io/fm-orchestrator/issue/333 - namespaces = ['rpms'] - version = RAWHIDE - else: - namespaces = ['rpms'] - if tag.startswith('epel'): - version = tag.split('epel')[1] - elif tag.startswith('f'): - version = tag.split('f')[1] - elif tag.endswith('epel') and tag.startswith('dist'): - # This is for older EPEL tags such as dit-6E-epel - version = tag.split('-')[1][:-1] - else: - print('Error: an invalid tag was specified', file=sys.stderr) - sys.exit(1) - - return namespaces, version - - -def get_branch_and_arches(tag, version): - if tag.startswith('epel'): - # Ex: epel7 => epel7 - branch = tag - arches = ["primary"] - elif tag.endswith('epel'): - # Ex: dist-6E-epel => el6 - branch = 'el%s' % version - arches = ["primary"] - elif tag == 'module-package-list': - branch = 'rawhide' - arches = ["primary"] - else: - # Fedora - if version == RAWHIDE: - branch = 'rawhide' - else: - branch = tag.split('-')[0] - - if STAGING: - arches = ["primary"] - else: - if version <= "26": - arches = ["primary", "s390"] - else: - # Yay! Everything in primary. - arches = ["primary"] - - return branch, arches - - -def get_active_releases_from_bodhi(): - bodhi_url = '{0}releases/?exclude_archived=True'.format(BODHI_URL) - - rv = requests.get(bodhi_url, timeout=60) - - if rv.ok: - active_releases = [] - rv_json = rv.json() - if rv_json['releases']: - for release in rv_json['releases']: - if re.match(r'^(f|epel)\d{1,2}$', release['branch']): - active_releases.append(release['branch']) - return list(set(active_releases)) - return [] - def get_project_branches(session, namespace, project_name): """ Returns list of branches for the repo from Pagure dist-git. @@ -222,8 +139,8 @@ def get_project_branches(session, namespace, project_name): return project_name, [] -def get_project_name_and_its_active_branches(session, namespace, active_releases, - lookaside, project_name, verbose=False): +def get_project_name_and_its_active_branches(session, namespace, active_branches, + retired_pkgs, project_name, verbose=False): """ Gets the branches on a project. This function is used for mapping. :param namespace: string of the namespace the project is in @@ -236,18 +153,19 @@ def get_project_name_and_its_active_branches(session, namespace, active_releases print('- Querying pagure distgit for package branches') project_branches = get_project_branches(session, namespace, project_name) try: - active_package_branches = list(set(active_releases) & set(project_branches)) + ['rawhide'] + active_package_branches = set(active_branches) & set(project_branches)) for branch in active_package_branches: - if project_name in lookaside[branch]: + if project_name in retired_pkgs[branch]: active_package_branches.remove(branch) return project_name, active_package_branches except TypeError: - print("One of the lists is probably empty: active_releases: {active_releases}, project_branches: {project_branches}") + print("One of the lists is probably empty: active_branches: {active_branches}, project_branches: {project_branches}") # Check if a package is not retired on any of the branches return project_name, [] + def get_pagure_project_names_from_page(session, namespace, page, package=None, verbose=False): """ @@ -279,7 +197,7 @@ def get_pagure_project_names_from_page(session, namespace, page, return set() -def get_pagure_project_branches(namespace, active_releases, lookaside, package=None, verbose=False): +def get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=None, verbose=False): """ Gets all the branches of all the Pagure projects in the desired namespace :param namespace: string of the namespace to query for projects @@ -322,7 +240,7 @@ def get_pagure_project_branches(namespace, active_releases, lookaside, package=N # Since we are going to multi-thread, we need to make a partial function # call so that all the function needs is an iterable to run partial_get_project_name_and_branch = partial( - get_project_name_and_its_active_branches, session, namespace, active_releases, lookaside, + get_project_name_and_its_active_branches, session, namespace, active_branches, retired_pkgs, verbose=verbose) # Get a list of tuples in the form of (project, [branch...]), then convert # that to a dictionary @@ -453,35 +371,58 @@ if __name__ == '__main__': tags = args.tag package = args.package - # Get all the info about the tags we are interested in + # Let's start with getting release info from bodhi + def get_page(page): + rv = requests.get( + BODHI_URL + 'releases/', + params={'exclude_archived': True, 'page': page} + ) + rv.raise_for_status() + rv_json = rv.json() + return rv_json['releases'], rv_json['pages'] + + releases, pages = get_page(1) + if pages > 1: + for page in range(2, pages + 1): + more_releases, _ = get_page(page) + releases.extend(more_releases) + + active_branches = set() + bodhi_releases = {} + for release in releases: + if release['dist_tag'] != 'eln': + active_branches.add(release['branch']) + bodhi_releases[release['dist_tag']] = release + + # Get all the info about the tags we are interested in, + # and also retired packages per branch unique_namespaces = set() tag_info = {} for tag in tags: - namespaces, version = get_namespaces_and_version_from_tag(tag) - branch, arches = get_branch_and_arches(tag, version) + if tag not in bodhi_releases: + raise SystemExit(f'no active bodhi releases use the {tag} tag') + + if 'container' in tag: + namespaces = ['container'] + elif 'flatpak' in tag: + namespaces = ['flatpaks'] + else: + namespaces = ['rpms'] + tag_info[tag] = { 'namespaces': namespaces, - 'version': version, - 'branch': branch, - 'arches': arches + 'version': bodhi_releases[tag]['version'], + 'branch': bodhi_releases[tag]['branch'], + 'arches': ['primary'], } unique_namespaces.update(namespaces) - # Let's start with getting the active releases from bodhi - active_releases = get_active_releases_from_bodhi() - # Let's fetch the json files with retired packages per release from lookaside cache - # This is a bit ugly, but the idea is to have the latest release removed in favor of rawhide - rawhide_active_releases = active_releases[:] - rawhide_active_releases.remove(max(rawhide_active_releases)) - rawhide_active_releases.append('rawhide') - # Let's store the json files with retired packages in lookaside - lookaside = {} - for branch in rawhide_active_releases: - url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(branch) + retired_pkgs = {} + for branch in active_branches: + url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(bodhi_releases[tag]['branch']) rv = requests.get(url) # change to session - lookaside[branch] = rv.json() - + retired_pkgs.update(rv.json()) # Get all the project to branch mappings for every namespace namespace_to_projects = {} @@ -490,7 +431,7 @@ if __name__ == '__main__': print('Querying for all the projects with the namespace "{0}"' .format(namespace)) namespace_to_projects[namespace] = \ - get_pagure_project_branches(namespace, active_releases, lookaside, package=package, verbose=verbose) + get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=package, verbose=verbose) for tag, info in list(tag_info.items()): if verbose: