Update owner-sync-pagure.j2 to work with EPEL 10
Signed-off-by: Carl George <carlwgeorge@gmail.com>
This commit is contained in:
Carl George
zlopez
parent
6ff3450ce8
commit
01be34a706
3 changed files with 53 additions and 111 deletions
|
|
@ -116,7 +116,7 @@
|
|||
# bodhi2/backend/files/koji_sync_listener.py
|
||||
# This cronjob runs only once a day. The listener script runs reactively.
|
||||
cron: name="owner-sync" minute="15" hour="4" user="root"
|
||||
job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel9 epel9-next epel8'"
|
||||
job="/usr/local/bin/lock-wrapper owner-sync '/usr/local/bin/owner-sync-pagure f41 f40 f39 f39-flatpak epel10.0 epel9 epel9-next epel8'"
|
||||
cron_file=update-koji-owner
|
||||
user=apache
|
||||
when: env == "production"
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ taglist = [
|
|||
"f39",
|
||||
"f39-container",
|
||||
"f39-flatpak",
|
||||
"epel10.0",
|
||||
"epel9",
|
||||
"epel9-next",
|
||||
"epel8",
|
||||
|
|
|
|||
|
|
@ -108,89 +108,6 @@ def get_options():
|
|||
return opts
|
||||
|
||||
|
||||
def get_namespaces_and_version_from_tag(tag):
|
||||
if 'container' in tag:
|
||||
namespaces = ['container']
|
||||
version = tag.split('-')[0].split('f')[1]
|
||||
elif 'docker' in tag:
|
||||
namespaces = ['container']
|
||||
version = tag.split('-')[0].split('f')[1]
|
||||
elif 'modular' in tag:
|
||||
namespaces = ['flatpaks', 'modules']
|
||||
try:
|
||||
version = tag.split('-')[0].split('f')[1]
|
||||
except IndexError:
|
||||
version = RAWHIDE
|
||||
elif 'flatpak' in tag:
|
||||
namespaces = ['flatpaks']
|
||||
version = tag.split('-')[0].split('f')[1]
|
||||
elif tag == 'module-package-list':
|
||||
# See https://pagure.io/releng/issue/6663
|
||||
# and https://pagure.io/fm-orchestrator/issue/333
|
||||
namespaces = ['rpms']
|
||||
version = RAWHIDE
|
||||
else:
|
||||
namespaces = ['rpms']
|
||||
if tag.startswith('epel'):
|
||||
version = tag.split('epel')[1]
|
||||
elif tag.startswith('f'):
|
||||
version = tag.split('f')[1]
|
||||
elif tag.endswith('epel') and tag.startswith('dist'):
|
||||
# This is for older EPEL tags such as dit-6E-epel
|
||||
version = tag.split('-')[1][:-1]
|
||||
else:
|
||||
print('Error: an invalid tag was specified', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
return namespaces, version
|
||||
|
||||
|
||||
def get_branch_and_arches(tag, version):
|
||||
if tag.startswith('epel'):
|
||||
# Ex: epel7 => epel7
|
||||
branch = tag
|
||||
arches = ["primary"]
|
||||
elif tag.endswith('epel'):
|
||||
# Ex: dist-6E-epel => el6
|
||||
branch = 'el%s' % version
|
||||
arches = ["primary"]
|
||||
elif tag == 'module-package-list':
|
||||
branch = 'rawhide'
|
||||
arches = ["primary"]
|
||||
else:
|
||||
# Fedora
|
||||
if version == RAWHIDE:
|
||||
branch = 'rawhide'
|
||||
else:
|
||||
branch = tag.split('-')[0]
|
||||
|
||||
if STAGING:
|
||||
arches = ["primary"]
|
||||
else:
|
||||
if version <= "26":
|
||||
arches = ["primary", "s390"]
|
||||
else:
|
||||
# Yay! Everything in primary.
|
||||
arches = ["primary"]
|
||||
|
||||
return branch, arches
|
||||
|
||||
|
||||
def get_active_releases_from_bodhi():
|
||||
bodhi_url = '{0}releases/?exclude_archived=True'.format(BODHI_URL)
|
||||
|
||||
rv = requests.get(bodhi_url, timeout=60)
|
||||
|
||||
if rv.ok:
|
||||
active_releases = []
|
||||
rv_json = rv.json()
|
||||
if rv_json['releases']:
|
||||
for release in rv_json['releases']:
|
||||
if re.match(r'^(f|epel)\d{1,2}$', release['branch']):
|
||||
active_releases.append(release['branch'])
|
||||
return list(set(active_releases))
|
||||
return []
|
||||
|
||||
def get_project_branches(session, namespace, project_name):
|
||||
"""
|
||||
Returns list of branches for the repo from Pagure dist-git.
|
||||
|
|
@ -222,8 +139,8 @@ def get_project_branches(session, namespace, project_name):
|
|||
return project_name, []
|
||||
|
||||
|
||||
def get_project_name_and_its_active_branches(session, namespace, active_releases,
|
||||
lookaside, project_name, verbose=False):
|
||||
def get_project_name_and_its_active_branches(session, namespace, active_branches,
|
||||
retired_pkgs, project_name, verbose=False):
|
||||
"""
|
||||
Gets the branches on a project. This function is used for mapping.
|
||||
:param namespace: string of the namespace the project is in
|
||||
|
|
@ -236,18 +153,19 @@ def get_project_name_and_its_active_branches(session, namespace, active_releases
|
|||
print('- Querying pagure distgit for package branches')
|
||||
project_branches = get_project_branches(session, namespace, project_name)
|
||||
try:
|
||||
active_package_branches = list(set(active_releases) & set(project_branches)) + ['rawhide']
|
||||
active_package_branches = set(active_branches) & set(project_branches))
|
||||
for branch in active_package_branches:
|
||||
if project_name in lookaside[branch]:
|
||||
if project_name in retired_pkgs[branch]:
|
||||
active_package_branches.remove(branch)
|
||||
|
||||
return project_name, active_package_branches
|
||||
|
||||
except TypeError:
|
||||
print("One of the lists is probably empty: active_releases: {active_releases}, project_branches: {project_branches}")
|
||||
print("One of the lists is probably empty: active_branches: {active_branches}, project_branches: {project_branches}")
|
||||
# Check if a package is not retired on any of the branches
|
||||
return project_name, []
|
||||
|
||||
|
||||
def get_pagure_project_names_from_page(session, namespace, page,
|
||||
package=None, verbose=False):
|
||||
"""
|
||||
|
|
@ -279,7 +197,7 @@ def get_pagure_project_names_from_page(session, namespace, page,
|
|||
return set()
|
||||
|
||||
|
||||
def get_pagure_project_branches(namespace, active_releases, lookaside, package=None, verbose=False):
|
||||
def get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=None, verbose=False):
|
||||
"""
|
||||
Gets all the branches of all the Pagure projects in the desired namespace
|
||||
:param namespace: string of the namespace to query for projects
|
||||
|
|
@ -322,7 +240,7 @@ def get_pagure_project_branches(namespace, active_releases, lookaside, package=N
|
|||
# Since we are going to multi-thread, we need to make a partial function
|
||||
# call so that all the function needs is an iterable to run
|
||||
partial_get_project_name_and_branch = partial(
|
||||
get_project_name_and_its_active_branches, session, namespace, active_releases, lookaside,
|
||||
get_project_name_and_its_active_branches, session, namespace, active_branches, retired_pkgs,
|
||||
verbose=verbose)
|
||||
# Get a list of tuples in the form of (project, [branch...]), then convert
|
||||
# that to a dictionary
|
||||
|
|
@ -453,35 +371,58 @@ if __name__ == '__main__':
|
|||
tags = args.tag
|
||||
package = args.package
|
||||
|
||||
# Get all the info about the tags we are interested in
|
||||
# Let's start with getting release info from bodhi
|
||||
def get_page(page):
|
||||
rv = requests.get(
|
||||
BODHI_URL + 'releases/',
|
||||
params={'exclude_archived': True, 'page': page}
|
||||
)
|
||||
rv.raise_for_status()
|
||||
rv_json = rv.json()
|
||||
return rv_json['releases'], rv_json['pages']
|
||||
|
||||
releases, pages = get_page(1)
|
||||
if pages > 1:
|
||||
for page in range(2, pages + 1):
|
||||
more_releases, _ = get_page(page)
|
||||
releases.extend(more_releases)
|
||||
|
||||
active_branches = set()
|
||||
bodhi_releases = {}
|
||||
for release in releases:
|
||||
if release['dist_tag'] != 'eln':
|
||||
active_branches.add(release['branch'])
|
||||
bodhi_releases[release['dist_tag']] = release
|
||||
|
||||
# Get all the info about the tags we are interested in,
|
||||
# and also retired packages per branch
|
||||
unique_namespaces = set()
|
||||
tag_info = {}
|
||||
for tag in tags:
|
||||
namespaces, version = get_namespaces_and_version_from_tag(tag)
|
||||
branch, arches = get_branch_and_arches(tag, version)
|
||||
if tag not in bodhi_releases:
|
||||
raise SystemExit(f'no active bodhi releases use the {tag} tag')
|
||||
|
||||
if 'container' in tag:
|
||||
namespaces = ['container']
|
||||
elif 'flatpak' in tag:
|
||||
namespaces = ['flatpaks']
|
||||
else:
|
||||
namespaces = ['rpms']
|
||||
|
||||
tag_info[tag] = {
|
||||
'namespaces': namespaces,
|
||||
'version': version,
|
||||
'branch': branch,
|
||||
'arches': arches
|
||||
'version': bodhi_releases[tag]['version'],
|
||||
'branch': bodhi_releases[tag]['branch'],
|
||||
'arches': ['primary'],
|
||||
}
|
||||
unique_namespaces.update(namespaces)
|
||||
|
||||
# Let's start with getting the active releases from bodhi
|
||||
active_releases = get_active_releases_from_bodhi()
|
||||
|
||||
# Let's fetch the json files with retired packages per release from lookaside cache
|
||||
# This is a bit ugly, but the idea is to have the latest release removed in favor of rawhide
|
||||
rawhide_active_releases = active_releases[:]
|
||||
rawhide_active_releases.remove(max(rawhide_active_releases))
|
||||
rawhide_active_releases.append('rawhide')
|
||||
# Let's store the json files with retired packages in lookaside
|
||||
lookaside = {}
|
||||
for branch in rawhide_active_releases:
|
||||
url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(branch)
|
||||
retired_pkgs = {}
|
||||
for branch in active_branches:
|
||||
url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(bodhi_releases[tag]['branch'])
|
||||
rv = requests.get(url) # change to session
|
||||
lookaside[branch] = rv.json()
|
||||
|
||||
retired_pkgs.update(rv.json())
|
||||
|
||||
# Get all the project to branch mappings for every namespace
|
||||
namespace_to_projects = {}
|
||||
|
|
@ -490,7 +431,7 @@ if __name__ == '__main__':
|
|||
print('Querying for all the projects with the namespace "{0}"'
|
||||
.format(namespace))
|
||||
namespace_to_projects[namespace] = \
|
||||
get_pagure_project_branches(namespace, active_releases, lookaside, package=package, verbose=verbose)
|
||||
get_pagure_project_branches(namespace, active_branches, retired_pkgs, package=package, verbose=verbose)
|
||||
|
||||
for tag, info in list(tag_info.items()):
|
||||
if verbose:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue