Remove pdc calls from owner-sync-pagure.j2 script
Signed-off-by: Lenka Segura <lsegura@redhat.com>
This commit is contained in:
parent
f2bfbb8cd7
commit
7fde6612e5
1 changed files with 75 additions and 19 deletions
|
@ -22,6 +22,7 @@ from urllib.parse import urljoin
|
||||||
import multiprocessing.pool
|
import multiprocessing.pool
|
||||||
from math import ceil
|
from math import ceil
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
import re
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
import koji
|
import koji
|
||||||
|
@ -50,6 +51,7 @@ STAGING = {{ 'True' if env == 'staging' else 'False' }}
|
||||||
HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org'
|
HOSTNAME = 'bodhi{{ env_suffix }}.fedoraproject.org'
|
||||||
IPA_REALM = '{{ ipa_realm }}'
|
IPA_REALM = '{{ ipa_realm }}'
|
||||||
ENV_SUFFIX = '{{ env_suffix }}'
|
ENV_SUFFIX = '{{ env_suffix }}'
|
||||||
|
BODHI_URL = 'https://bodhi.fedoraproject.org/'
|
||||||
if STAGING:
|
if STAGING:
|
||||||
PAGURE_URL = 'https://src.stg.fedoraproject.org/'
|
PAGURE_URL = 'https://src.stg.fedoraproject.org/'
|
||||||
PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
|
PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
|
||||||
|
@ -186,8 +188,52 @@ def get_branch_and_arches(tag, version):
|
||||||
return branch, arches
|
return branch, arches
|
||||||
|
|
||||||
|
|
||||||
def get_pdc_project_name_and_branch(session, namespace, project_name,
|
def get_active_releases_from_bodhi():
|
||||||
verbose=False):
|
bodhi_url = '{0}releases/?exclude_archived=True'.format(BODHI_URL)
|
||||||
|
|
||||||
|
rv = requests.get(bodhi_url, timeout=60)
|
||||||
|
|
||||||
|
if not rv.ok:
|
||||||
|
return []
|
||||||
|
|
||||||
|
active_releases = []
|
||||||
|
rv_json = rv.json()
|
||||||
|
if rv_json['releases']:
|
||||||
|
for release in rv_json['releases']:
|
||||||
|
if re.match(r'^(f|epel)\d{1,2}$', release['branch']):
|
||||||
|
active_releases.append(release['branch'])
|
||||||
|
return list(set(active_releases))
|
||||||
|
|
||||||
|
|
||||||
|
def get_project_branches(session, namespace, project_name):
|
||||||
|
"""
|
||||||
|
Returns list of branches for the repo from Pagure dist-git.
|
||||||
|
:param logger: A logger object
|
||||||
|
:param url: a string of the URL to Pagure
|
||||||
|
:param namespace: a string determines a type of the repository
|
||||||
|
:param repo_name: a string of the repository name
|
||||||
|
:return: a list of branches
|
||||||
|
"""
|
||||||
|
get_branches_url = '{0}api/0/{1}/{2}/git/branches'.format(PAGURE_URL, namespace, project_name)
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
'Accept': 'application/json',
|
||||||
|
'Content-Type': 'application/json'
|
||||||
|
}
|
||||||
|
rv = requests.get(
|
||||||
|
get_branches_url, headers=headers, timeout=60)
|
||||||
|
rv_json = rv.json()
|
||||||
|
|
||||||
|
if not rv.ok:
|
||||||
|
# When specific namespace has no branches, API returns error "Project not found".
|
||||||
|
# Do not fail. Return "no branches found" instead.
|
||||||
|
return project_name, []
|
||||||
|
|
||||||
|
return rv_json.get("branches", ())
|
||||||
|
|
||||||
|
|
||||||
|
def get_project_name_and_its_active_branches(session, namespace, active_releases,
|
||||||
|
lookaside, project_name, verbose=False):
|
||||||
"""
|
"""
|
||||||
Gets the branches on a project. This function is used for mapping.
|
Gets the branches on a project. This function is used for mapping.
|
||||||
:param namespace: string of the namespace the project is in
|
:param namespace: string of the namespace the project is in
|
||||||
|
@ -196,24 +242,18 @@ def get_pdc_project_name_and_branch(session, namespace, project_name,
|
||||||
:return: a tuple containing the string of the project and a list of
|
:return: a tuple containing the string of the project and a list of
|
||||||
branches
|
branches
|
||||||
"""
|
"""
|
||||||
project_branches_url = '{0}component-branches/'.format(PDC_URL)
|
|
||||||
params = dict(
|
|
||||||
global_component=project_name,
|
|
||||||
type=PDC_TYPES[namespace],
|
|
||||||
active=True,
|
|
||||||
)
|
|
||||||
if verbose:
|
if verbose:
|
||||||
print('- Querying {0} {1}'.format(project_branches_url, params))
|
print('- Querying pagure distgit for package branches')
|
||||||
project_branches_rv = session.get(
|
project_branches = get_project_branches(session, namespace, project_name)
|
||||||
project_branches_url, params=params, verify=VERIFY, timeout=60)
|
|
||||||
|
|
||||||
# If the project's branches can't be reported, just return no branches and
|
active_package_branches = list(set(active_releases) & set(project_branches)) + ['rawhide']
|
||||||
# it will be skipped later on
|
|
||||||
if not project_branches_rv.ok:
|
|
||||||
return project_name, []
|
|
||||||
|
|
||||||
data = project_branches_rv.json()
|
# Check if a package is not retired on any of the branches
|
||||||
return project_name, [branch['name'] for branch in data['results']]
|
for branch in active_package_branches:
|
||||||
|
if project_name in lookaside[branch]:
|
||||||
|
active_package_branches.remove(branch)
|
||||||
|
|
||||||
|
return project_name, active_package_branches
|
||||||
|
|
||||||
|
|
||||||
def get_pagure_project_names_from_page(session, namespace, page,
|
def get_pagure_project_names_from_page(session, namespace, page,
|
||||||
|
@ -288,7 +328,7 @@ def get_pagure_project_branches(namespace, package=None, verbose=False):
|
||||||
# Since we are going to multi-thread, we need to make a partial function
|
# Since we are going to multi-thread, we need to make a partial function
|
||||||
# call so that all the function needs is an iterable to run
|
# call so that all the function needs is an iterable to run
|
||||||
partial_get_pdc_project_name_and_branch = partial(
|
partial_get_pdc_project_name_and_branch = partial(
|
||||||
get_pdc_project_name_and_branch, session, namespace,
|
get_project_name_and_its_active_branches, session, namespace, active_releases, lookaside,
|
||||||
verbose=verbose)
|
verbose=verbose)
|
||||||
# Get a list of tuples in the form of (project, [branch...]), then convert
|
# Get a list of tuples in the form of (project, [branch...]), then convert
|
||||||
# that to a dictionary
|
# that to a dictionary
|
||||||
|
@ -433,6 +473,22 @@ if __name__ == '__main__':
|
||||||
}
|
}
|
||||||
unique_namespaces.update(namespaces)
|
unique_namespaces.update(namespaces)
|
||||||
|
|
||||||
|
# Let's start with getting the active releases from bodhi
|
||||||
|
active_releases = get_active_releases_from_bodhi()
|
||||||
|
|
||||||
|
# Let's fetch the json files with retired packages per release from lookaside cache
|
||||||
|
# This is a bit ugly, but the idea is to have the latest release removed in favor of rawhide
|
||||||
|
rawhide_active_releases = active_releases[:]
|
||||||
|
rawhide_active_releases.remove(max(rawhide_active_releases))
|
||||||
|
rawhide_active_releases + ['rawhide']
|
||||||
|
# Let's store the json files with retired packages in lookaside
|
||||||
|
lookaside = {}
|
||||||
|
for branch in rawhide_active_releases:
|
||||||
|
url = "https://src.fedoraproject.org/lookaside/retired_in_{0}.json".format(branch)
|
||||||
|
rv = requests.get(url) # change to session
|
||||||
|
lookaside[branch] = rv.json()
|
||||||
|
|
||||||
|
|
||||||
# Get all the project to branch mappings for every namespace
|
# Get all the project to branch mappings for every namespace
|
||||||
namespace_to_projects = {}
|
namespace_to_projects = {}
|
||||||
for namespace in unique_namespaces:
|
for namespace in unique_namespaces:
|
||||||
|
@ -440,7 +496,7 @@ if __name__ == '__main__':
|
||||||
print('Querying for all the projects with the namespace "{0}"'
|
print('Querying for all the projects with the namespace "{0}"'
|
||||||
.format(namespace))
|
.format(namespace))
|
||||||
namespace_to_projects[namespace] = \
|
namespace_to_projects[namespace] = \
|
||||||
get_pagure_project_branches(namespace, package=package, verbose=verbose)
|
get_pagure_project_branches(namespace, active_releases, lookaside, package=package, verbose=verbose)
|
||||||
|
|
||||||
for tag, info in list(tag_info.items()):
|
for tag, info in list(tag_info.items()):
|
||||||
if verbose:
|
if verbose:
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue