Update owner-sync-pagure.j2 to work with EPEL 10

Signed-off-by: Carl George <carlwgeorge@gmail.com>

roles/bodhi2/backend/tasks/main.yml

@@ -116,7 +116,7 @@
   # bodhi2/backend/files/koji_sync_listener.py
   # This cronjob runs only once a day.  The listener script runs reactively.
   cron: name="owner-sync" minute="15" hour="4" user="root"
-      job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel9 epel9-next epel8'"
+      job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel10.0 epel9 epel9-next epel8'"
       cron_file=update-koji-owner
       user=apache
   when: env == "production"

roles/bodhi2/backend/templates/koji_sync_listener.toml

@@ -45,6 +45,7 @@ taglist = [
     "f39",
     "f39-container",
     "f39-flatpak",
+    "epel10.0",
     "epel9",
     "epel9-next",
     "epel8",

roles/bodhi2/backend/templates/owner-sync-pagure.j2

@@ -108,89 +108,6 @@ def get_options():
     return opts
 
 
-def get_namespaces_and_version_from_tag(tag):
-    if 'container' in tag:
-        namespaces = ['container']
-        version = tag.split('-')[0].split('f')[1]
-    elif 'docker' in tag:
-        namespaces = ['container']
-        version = tag.split('-')[0].split('f')[1]
-    elif 'modular' in tag:
-        namespaces = ['flatpaks', 'modules']
-        try:
-            version = tag.split('-')[0].split('f')[1]
-        except IndexError:
-            version = RAWHIDE
-    elif 'flatpak' in tag:
-        namespaces = ['flatpaks']
-        version = tag.split('-')[0].split('f')[1]
-    elif tag == 'module-package-list':
-        # See https://pagure.io/releng/issue/6663
-        # and https://pagure.io/fm-orchestrator/issue/333
-        namespaces = ['rpms']
-        version = RAWHIDE
-    else:
-        namespaces = ['rpms']
-        if tag.startswith('epel'):
-            version = tag.split('epel')[1]
-        elif tag.startswith('f'):
-            version = tag.split('f')[1]
-        elif tag.endswith('epel') and tag.startswith('dist'):
-            # This is for older EPEL tags such as dit-6E-epel
-            version = tag.split('-')[1][:-1]
-        else:
-            print('Error: an invalid tag was specified', file=sys.stderr)
-            sys.exit(1)
-
-    return namespaces, version
-
-
-def get_branch_and_arches(tag, version):
-    if tag.startswith('epel'):
-        # Ex: epel7 => epel7
-        branch = tag
-        arches = ["primary"]
-    elif tag.endswith('epel'):
-        # Ex: dist-6E-epel => el6
-        branch = 'el%s' % version
-        arches = ["primary"]
-    elif tag == 'module-package-list':
-        branch = 'rawhide'
-        arches = ["primary"]
-    else:
-        # Fedora
-        if version == RAWHIDE:
-            branch = 'rawhide'
-        else:
-            branch = tag.split('-')[0]
-
-        if STAGING:
-            arches = ["primary"]
-        else:
-            if version <= "26":
-                arches = ["primary", "s390"]
-            else:
-                # Yay!  Everything in primary.
-                arches = ["primary"]
-
-    return branch, arches
-
-
-def get_active_releases_from_bodhi():
-    bodhi_url = '{0}releases/?exclude_archived=True'.format(BODHI_URL)
-
-    rv = requests.get(bodhi_url, timeout=60)
-
-    if rv.ok:
-        active_releases = []
-        rv_json = rv.json()
-        if rv_json['releases']:
-            for release in rv_json['releases']:
-                if re.match(r'^(f|epel)\d{1,2}$', release['branch']):
-                    active_releases.append(release['branch'])
-        return list(set(active_releases))
-    return []
-
 def get_project_branches(session, namespace, project_name):
     """
     Returns list of branches for the repo from Pagure dist-git.
@@ -222,8 +139,8 @@ def get_project_branches(session, namespace, project_name):
     return project_name, []
 
 
-def get_project_name_and_its_active_branches(session, namespace, active_releases,
+def get_project_name_and_its_active_branches(session, namespace, active_branches,
-                                             lookaside, project_name, verbose=False):
+                                             retired_pkgs, project_name, verbose=False):
     """
     Gets the branches on a project. This function is used for mapping.
     :param namespace: string of the namespace the project is in
@@ -236,18 +153,19 @@ def get_project_name_and_its_active_branches(session, namespace, active_releases
         print('- Querying pagure distgit for package branches')
     project_branches = get_project_branches(session, namespace, project_name)
     try:
-        active_package_branches = list(set(active_releases) & set(project_branches)) + ['rawhide']
+        active_package_branches = set(active_branches) & set(project_branches))
         for branch in active_package_branches:
-            if project_name in lookaside[branch]:
+            if project_name in retired_pkgs[branch]:
                 active_package_branches.remove(branch)
 
         return project_name, active_package_branches
 
     except TypeError:
-        print("One of the lists is probably empty: active_releases: {active_releases}, project_branches: {project_branches}")
+        print("One of the lists is probably empty: active_branches: {active_branches}, project_branches: {project_branches}")
         # Check if a package is not retired on any of the branches
     return project_name, []
 
+
 def get_pagure_project_names_from_page(session, namespace, page,
                                        package=None, verbose=False):
     """
@@ -279,7 +197,7 @@ def get_pagure_project_names_from_page(session, namespace, page,
         return set()
 
 
-def get_pagure_project_branches(namespace, active_releases, lookaside, package=None, verbose=False):
+def get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=None, verbose=False):
     """
     Gets all the branches of all the Pagure projects in the desired namespace
     :param namespace: string of the namespace to query for projects
@@ -322,7 +240,7 @@ def get_pagure_project_branches(namespace, active_releases, lookaside, package=N
     # Since we are going to multi-thread, we need to make a partial function
     # call so that all the function needs is an iterable to run
     partial_get_project_name_and_branch = partial(
-        get_project_name_and_its_active_branches, session, namespace, active_releases, lookaside,
+        get_project_name_and_its_active_branches, session, namespace, active_branches, retired_pkgs,
         verbose=verbose)
     # Get a list of tuples in the form of (project, [branch...]), then convert
     # that to a dictionary
@@ -453,35 +371,58 @@ if __name__ == '__main__':
     tags = args.tag
     package = args.package
 
-    # Get all the info about the tags we are interested in
+    # Let's start with getting release info from bodhi
+    def get_page(page):
+        rv = requests.get(
+            BODHI_URL + 'releases/',
+            params={'exclude_archived': True, 'page': page}
+        )
+        rv.raise_for_status()
+        rv_json = rv.json()
+        return rv_json['releases'], rv_json['pages']
+
+    releases, pages = get_page(1)
+    if pages > 1:
+        for page in range(2, pages + 1):
+            more_releases, _ = get_page(page)
+            releases.extend(more_releases)
+
+    active_branches = set()
+    bodhi_releases = {}
+    for release in releases:
+        if release['dist_tag'] != 'eln':
+            active_branches.add(release['branch'])
+            bodhi_releases[release['dist_tag']] = release
+
+    # Get all the info about the tags we are interested in,
+    # and also retired packages per branch
     unique_namespaces = set()
     tag_info = {}
     for tag in tags:
-        namespaces, version = get_namespaces_and_version_from_tag(tag)
+        if tag not in bodhi_releases:
-        branch, arches = get_branch_and_arches(tag, version)
+            raise SystemExit(f'no active bodhi releases use the {tag} tag')
+
+        if 'container' in tag:
+            namespaces = ['container']
+        elif 'flatpak' in tag:
+            namespaces = ['flatpaks']
+        else:
+            namespaces = ['rpms']
+
         tag_info[tag] = {
             'namespaces': namespaces,
-            'version': version,
+            'version': bodhi_releases[tag]['version'],
-            'branch': branch,
+            'branch': bodhi_releases[tag]['branch'],
-            'arches': arches
+            'arches': ['primary'],
         }
         unique_namespaces.update(namespaces)
 
-    # Let's start with getting the active releases from bodhi
-    active_releases = get_active_releases_from_bodhi()
-
     # Let's fetch the json files with retired packages per release from lookaside cache
-    # This is a bit ugly, but the idea is to have the latest release removed in favor of rawhide
+    retired_pkgs = {}
-    rawhide_active_releases = active_releases[:]
+    for branch in active_branches:
-    rawhide_active_releases.remove(max(rawhide_active_releases))
+        url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(bodhi_releases[tag]['branch'])
-    rawhide_active_releases.append('rawhide')
-    # Let's store the json files with retired packages in lookaside
-    lookaside = {}
-    for branch in rawhide_active_releases:
-        url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(branch)
         rv = requests.get(url)  # change to session
-        lookaside[branch] = rv.json()
+        retired_pkgs.update(rv.json())
-
 
     # Get all the project to branch mappings for every namespace
     namespace_to_projects = {}
@@ -490,7 +431,7 @@ if __name__ == '__main__':
             print('Querying for all the projects with the namespace "{0}"'
                   .format(namespace))
         namespace_to_projects[namespace] = \
-            get_pagure_project_branches(namespace, active_releases, lookaside, package=package, verbose=verbose)
+            get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=package, verbose=verbose)
 
     for tag, info in list(tag_info.items()):
         if verbose: