Remove pdc calls from owner-sync-pagure.j2 script

Signed-off-by: Lenka Segura <lsegura@redhat.com>
This commit is contained in:
Lenka Segura 2024-04-11 23:31:15 +02:00 committed by zlopez
parent f2bfbb8cd7
commit 7fde6612e5

View file

@ -22,6 +22,7 @@ from urllib.parse import urljoin
import multiprocessing.pool import multiprocessing.pool
from math import ceil from math import ceil
from functools import partial from functools import partial
import re
import requests import requests
import koji import koji
@ -50,6 +51,7 @@ STAGING = {{ 'True' if env == 'staging' else 'False' }}
HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org' HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org'
IPA_REALM = '{{ ipa_realm }}' IPA_REALM = '{{ ipa_realm }}'
ENV_SUFFIX = '{{ env_suffix }}' ENV_SUFFIX = '{{ env_suffix }}'
BODHI_URL = 'https://bodhi.fedoraproject.org/'
if STAGING: if STAGING:
PAGURE_URL = 'https://src.stg.fedoraproject.org/' PAGURE_URL = 'https://src.stg.fedoraproject.org/'
PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/' PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
@ -186,8 +188,52 @@ def get_branch_and_arches(tag, version):
return branch, arches return branch, arches
def get_pdc_project_name_and_branch(session, namespace, project_name, def get_active_releases_from_bodhi():
verbose=False): bodhi_url = '{0}releases/?exclude_archived=True'.format(BODHI_URL)
rv = requests.get(bodhi_url, timeout=60)
if not rv.ok:
return []
active_releases = []
rv_json = rv.json()
if rv_json['releases']:
for release in rv_json['releases']:
if re.match(r'^(f|epel)\d{1,2}$', release['branch']):
active_releases.append(release['branch'])
return list(set(active_releases))
def get_project_branches(session, namespace, project_name):
"""
Returns list of branches for the repo from Pagure dist-git.
:param logger: A logger object
:param url: a string of the URL to Pagure
:param namespace: a string determines a type of the repository
:param repo_name: a string of the repository name
:return: a list of branches
"""
get_branches_url = '{0}api/0/{1}/{2}/git/branches'.format(PAGURE_URL, namespace, project_name)
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
rv = requests.get(
get_branches_url, headers=headers, timeout=60)
rv_json = rv.json()
if not rv.ok:
# When specific namespace has no branches, API returns error "Project not found".
# Do not fail. Return "no branches found" instead.
return project_name, []
return rv_json.get("branches", ())
def get_project_name_and_its_active_branches(session, namespace, active_releases,
lookaside, project_name, verbose=False):
""" """
Gets the branches on a project. This function is used for mapping. Gets the branches on a project. This function is used for mapping.
:param namespace: string of the namespace the project is in :param namespace: string of the namespace the project is in
@ -196,24 +242,18 @@ def get_pdc_project_name_and_branch(session, namespace, project_name,
:return: a tuple containing the string of the project and a list of :return: a tuple containing the string of the project and a list of
branches branches
""" """
project_branches_url = '{0}component-branches/'.format(PDC_URL)
params = dict(
global_component=project_name,
type=PDC_TYPES[namespace],
active=True,
)
if verbose: if verbose:
print('- Querying {0} {1}'.format(project_branches_url, params)) print('- Querying pagure distgit for package branches')
project_branches_rv = session.get( project_branches = get_project_branches(session, namespace, project_name)
project_branches_url, params=params, verify=VERIFY, timeout=60)
# If the project's branches can't be reported, just return no branches and active_package_branches = list(set(active_releases) & set(project_branches)) + ['rawhide']
# it will be skipped later on
if not project_branches_rv.ok:
return project_name, []
data = project_branches_rv.json() # Check if a package is not retired on any of the branches
return project_name, [branch['name'] for branch in data['results']] for branch in active_package_branches:
if project_name in lookaside[branch]:
active_package_branches.remove(branch)
return project_name, active_package_branches
def get_pagure_project_names_from_page(session, namespace, page, def get_pagure_project_names_from_page(session, namespace, page,
@ -288,7 +328,7 @@ def get_pagure_project_branches(namespace, package=None, verbose=False):
# Since we are going to multi-thread, we need to make a partial function # Since we are going to multi-thread, we need to make a partial function
# call so that all the function needs is an iterable to run # call so that all the function needs is an iterable to run
partial_get_pdc_project_name_and_branch = partial( partial_get_pdc_project_name_and_branch = partial(
get_pdc_project_name_and_branch, session, namespace, get_project_name_and_its_active_branches, session, namespace, active_releases, lookaside,
verbose=verbose) verbose=verbose)
# Get a list of tuples in the form of (project, [branch...]), then convert # Get a list of tuples in the form of (project, [branch...]), then convert
# that to a dictionary # that to a dictionary
@ -433,6 +473,22 @@ if __name__ == '__main__':
} }
unique_namespaces.update(namespaces) unique_namespaces.update(namespaces)
# Let's start with getting the active releases from bodhi
active_releases = get_active_releases_from_bodhi()
# Let's fetch the json files with retired packages per release from lookaside cache
# This is a bit ugly, but the idea is to have the latest release removed in favor of rawhide
rawhide_active_releases = active_releases[:]
rawhide_active_releases.remove(max(rawhide_active_releases))
rawhide_active_releases + ['rawhide']
# Let's store the json files with retired packages in lookaside
lookaside = {}
for branch in rawhide_active_releases:
url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(branch)
rv = requests.get(url) # change to session
lookaside[branch] = rv.json()
# Get all the project to branch mappings for every namespace # Get all the project to branch mappings for every namespace
namespace_to_projects = {} namespace_to_projects = {}
for namespace in unique_namespaces: for namespace in unique_namespaces:
@ -440,7 +496,7 @@ if __name__ == '__main__':
print('Querying for all the projects with the namespace "{0}"' print('Querying for all the projects with the namespace "{0}"'
.format(namespace)) .format(namespace))
namespace_to_projects[namespace] = \ namespace_to_projects[namespace] = \
get_pagure_project_branches(namespace, package=package, verbose=verbose) get_pagure_project_branches(namespace, active_releases, lookaside, package=package, verbose=verbose)
for tag, info in list(tag_info.items()): for tag, info in list(tag_info.items()):
if verbose: if verbose: