Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Use requests retry mechanism in the bugzilla sync script.

Browse source
This commit is contained in:
Ralph Bean 2017-08-09 14:09:56 +00:00
parent 27c8f24930
commit 6b67be1f42
1 changed files with 27 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

33
roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2
View file

@ -50,6 +50,24 @@ import yaml
from six import string_types
from fedora.client.fas2 import AccountSystem
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
def retry_session():
session = requests.Session()
retry = Retry(
total=5,
read=5,
connect=5,
backoff_factor=0.3,
status_forcelist=(500, 502, 504),
)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
BZSERVER = 'https://bugzilla.redhat.com'
BZUSER = '{{ bugzilla_user }}'
BZPASS = '{{ bugzilla_password }}'
@ -436,6 +454,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
:param pagure_project: a dictionary of the JSON of a Pagure project
:return: a dictionary of the content that the Bugzilla API would output
"""
session = retry_session()
base_error_msg = ('The connection to "{0}" failed with the status code '
'{1} and output "{2}"')
watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format(
@ -443,7 +462,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
pagure_project['name'])
if DRY_RUN:
print('Querying {0}'.format(watchers_api_url))
watchers_rv = requests.get(watchers_api_url, timeout=60)
watchers_rv = session.get(watchers_api_url, timeout=60)
if not watchers_rv.ok:
error_msg = base_error_msg.format(
watchers_api_url, watchers_rv.status_code, watchers_rv.text)
@ -462,7 +481,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
MDAPIURL.rstrip('/'), pagure_project['name'])
if DRY_RUN:
print('Querying {0}'.format(mdapi_url))
mdapi_rv = requests.get(mdapi_url, timeout=60)
mdapi_rv = session.get(mdapi_url, timeout=60)
if mdapi_rv.ok:
mdapi_rv_json = mdapi_rv.json()
summary = mdapi_rv_json['summary']
@ -477,7 +496,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
project['name'])
override_rv = requests.get(pagure_override_url, timeout=30)
override_rv = session.get(pagure_override_url, timeout=30)
if override_rv.status_code == 200:
override_yaml = yaml.load(override_rv.text)
override_yaml = override_yaml.get('bugzilla_contact', {})
@ -526,15 +545,17 @@ if __name__ == '__main__':
pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&'
'per_page=100'.format(
PAGURE_DIST_GIT_URL.rstrip('/')))
session = retry_session()
while True:
if DRY_RUN:
print('Querying {0}'.format(pagure_rpms_api_url))
rv_json = requests.get(pagure_rpms_api_url, timeout=120).json()
rv_json = session.get(pagure_rpms_api_url, timeout=120).json()
for project in rv_json['projects']:
pagure_project_branches_api_url = (
'{0}/api/0/rpms/{1}/git/branches'
.format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name']))
branch_rv_json = requests.get(
branch_rv_json = session.get(
pagure_project_branches_api_url, timeout=60).json()
epel = False
fedora = False
@ -562,7 +583,7 @@ if __name__ == '__main__':
while True:
if DRY_RUN:
print('Querying {0}'.format(pagure_container_api_url))
rv_json = requests.get(pagure_container_api_url, timeout=120).json()
rv_json = session.get(pagure_container_api_url, timeout=120).json()
for project in rv_json['projects']:
project_pkgdb_schema = pagure_project_to_acl_schema(project)
projects_dict['Fedora Container'][project['name']] = \