diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 630455e91c..863bcd6542 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -50,6 +50,24 @@ import yaml from six import string_types from fedora.client.fas2 import AccountSystem +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + + +def retry_session(): + session = requests.Session() + retry = Retry( + total=5, + read=5, + connect=5, + backoff_factor=0.3, + status_forcelist=(500, 502, 504), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + BZSERVER = 'https://bugzilla.redhat.com' BZUSER = '{{ bugzilla_user }}' BZPASS = '{{ bugzilla_password }}' @@ -436,6 +454,7 @@ def pagure_project_to_acl_schema(pagure_project, product): :param pagure_project: a dictionary of the JSON of a Pagure project :return: a dictionary of the content that the Bugzilla API would output """ + session = retry_session() base_error_msg = ('The connection to "{0}" failed with the status code ' '{1} and output "{2}"') watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format( @@ -443,7 +462,7 @@ def pagure_project_to_acl_schema(pagure_project, product): pagure_project['name']) if DRY_RUN: print('Querying {0}'.format(watchers_api_url)) - watchers_rv = requests.get(watchers_api_url, timeout=60) + watchers_rv = session.get(watchers_api_url, timeout=60) if not watchers_rv.ok: error_msg = base_error_msg.format( watchers_api_url, watchers_rv.status_code, watchers_rv.text) @@ -462,7 +481,7 @@ def pagure_project_to_acl_schema(pagure_project, product): MDAPIURL.rstrip('/'), pagure_project['name']) if DRY_RUN: print('Querying {0}'.format(mdapi_url)) - mdapi_rv = requests.get(mdapi_url, timeout=60) + mdapi_rv = session.get(mdapi_url, timeout=60) if mdapi_rv.ok: mdapi_rv_json = mdapi_rv.json() summary = mdapi_rv_json['summary'] @@ -477,7 +496,7 @@ def pagure_project_to_acl_schema(pagure_project, product): PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'], project['name']) - override_rv = requests.get(pagure_override_url, timeout=30) + override_rv = session.get(pagure_override_url, timeout=30) if override_rv.status_code == 200: override_yaml = yaml.load(override_rv.text) override_yaml = override_yaml.get('bugzilla_contact', {}) @@ -526,15 +545,17 @@ if __name__ == '__main__': pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&' 'per_page=100'.format( PAGURE_DIST_GIT_URL.rstrip('/'))) + session = retry_session() + while True: if DRY_RUN: print('Querying {0}'.format(pagure_rpms_api_url)) - rv_json = requests.get(pagure_rpms_api_url, timeout=120).json() + rv_json = session.get(pagure_rpms_api_url, timeout=120).json() for project in rv_json['projects']: pagure_project_branches_api_url = ( '{0}/api/0/rpms/{1}/git/branches' .format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name'])) - branch_rv_json = requests.get( + branch_rv_json = session.get( pagure_project_branches_api_url, timeout=60).json() epel = False fedora = False @@ -562,7 +583,7 @@ if __name__ == '__main__': while True: if DRY_RUN: print('Querying {0}'.format(pagure_container_api_url)) - rv_json = requests.get(pagure_container_api_url, timeout=120).json() + rv_json = session.get(pagure_container_api_url, timeout=120).json() for project in rv_json['projects']: project_pkgdb_schema = pagure_project_to_acl_schema(project) projects_dict['Fedora Container'][project['name']] = \