Use requests retry mechanism in the bugzilla sync script.
This commit is contained in:
parent
27c8f24930
commit
6b67be1f42
1 changed files with 27 additions and 6 deletions
|
@ -50,6 +50,24 @@ import yaml
|
||||||
from six import string_types
|
from six import string_types
|
||||||
from fedora.client.fas2 import AccountSystem
|
from fedora.client.fas2 import AccountSystem
|
||||||
|
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from requests.packages.urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
|
||||||
|
def retry_session():
|
||||||
|
session = requests.Session()
|
||||||
|
retry = Retry(
|
||||||
|
total=5,
|
||||||
|
read=5,
|
||||||
|
connect=5,
|
||||||
|
backoff_factor=0.3,
|
||||||
|
status_forcelist=(500, 502, 504),
|
||||||
|
)
|
||||||
|
adapter = HTTPAdapter(max_retries=retry)
|
||||||
|
session.mount('http://', adapter)
|
||||||
|
session.mount('https://', adapter)
|
||||||
|
return session
|
||||||
|
|
||||||
BZSERVER = 'https://bugzilla.redhat.com'
|
BZSERVER = 'https://bugzilla.redhat.com'
|
||||||
BZUSER = '{{ bugzilla_user }}'
|
BZUSER = '{{ bugzilla_user }}'
|
||||||
BZPASS = '{{ bugzilla_password }}'
|
BZPASS = '{{ bugzilla_password }}'
|
||||||
|
@ -436,6 +454,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
|
||||||
:param pagure_project: a dictionary of the JSON of a Pagure project
|
:param pagure_project: a dictionary of the JSON of a Pagure project
|
||||||
:return: a dictionary of the content that the Bugzilla API would output
|
:return: a dictionary of the content that the Bugzilla API would output
|
||||||
"""
|
"""
|
||||||
|
session = retry_session()
|
||||||
base_error_msg = ('The connection to "{0}" failed with the status code '
|
base_error_msg = ('The connection to "{0}" failed with the status code '
|
||||||
'{1} and output "{2}"')
|
'{1} and output "{2}"')
|
||||||
watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format(
|
watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format(
|
||||||
|
@ -443,7 +462,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
|
||||||
pagure_project['name'])
|
pagure_project['name'])
|
||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
print('Querying {0}'.format(watchers_api_url))
|
print('Querying {0}'.format(watchers_api_url))
|
||||||
watchers_rv = requests.get(watchers_api_url, timeout=60)
|
watchers_rv = session.get(watchers_api_url, timeout=60)
|
||||||
if not watchers_rv.ok:
|
if not watchers_rv.ok:
|
||||||
error_msg = base_error_msg.format(
|
error_msg = base_error_msg.format(
|
||||||
watchers_api_url, watchers_rv.status_code, watchers_rv.text)
|
watchers_api_url, watchers_rv.status_code, watchers_rv.text)
|
||||||
|
@ -462,7 +481,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
|
||||||
MDAPIURL.rstrip('/'), pagure_project['name'])
|
MDAPIURL.rstrip('/'), pagure_project['name'])
|
||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
print('Querying {0}'.format(mdapi_url))
|
print('Querying {0}'.format(mdapi_url))
|
||||||
mdapi_rv = requests.get(mdapi_url, timeout=60)
|
mdapi_rv = session.get(mdapi_url, timeout=60)
|
||||||
if mdapi_rv.ok:
|
if mdapi_rv.ok:
|
||||||
mdapi_rv_json = mdapi_rv.json()
|
mdapi_rv_json = mdapi_rv.json()
|
||||||
summary = mdapi_rv_json['summary']
|
summary = mdapi_rv_json['summary']
|
||||||
|
@ -477,7 +496,7 @@ def pagure_project_to_acl_schema(pagure_project, product):
|
||||||
PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
|
PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
|
||||||
project['name'])
|
project['name'])
|
||||||
|
|
||||||
override_rv = requests.get(pagure_override_url, timeout=30)
|
override_rv = session.get(pagure_override_url, timeout=30)
|
||||||
if override_rv.status_code == 200:
|
if override_rv.status_code == 200:
|
||||||
override_yaml = yaml.load(override_rv.text)
|
override_yaml = yaml.load(override_rv.text)
|
||||||
override_yaml = override_yaml.get('bugzilla_contact', {})
|
override_yaml = override_yaml.get('bugzilla_contact', {})
|
||||||
|
@ -526,15 +545,17 @@ if __name__ == '__main__':
|
||||||
pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&'
|
pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&'
|
||||||
'per_page=100'.format(
|
'per_page=100'.format(
|
||||||
PAGURE_DIST_GIT_URL.rstrip('/')))
|
PAGURE_DIST_GIT_URL.rstrip('/')))
|
||||||
|
session = retry_session()
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
print('Querying {0}'.format(pagure_rpms_api_url))
|
print('Querying {0}'.format(pagure_rpms_api_url))
|
||||||
rv_json = requests.get(pagure_rpms_api_url, timeout=120).json()
|
rv_json = session.get(pagure_rpms_api_url, timeout=120).json()
|
||||||
for project in rv_json['projects']:
|
for project in rv_json['projects']:
|
||||||
pagure_project_branches_api_url = (
|
pagure_project_branches_api_url = (
|
||||||
'{0}/api/0/rpms/{1}/git/branches'
|
'{0}/api/0/rpms/{1}/git/branches'
|
||||||
.format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name']))
|
.format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name']))
|
||||||
branch_rv_json = requests.get(
|
branch_rv_json = session.get(
|
||||||
pagure_project_branches_api_url, timeout=60).json()
|
pagure_project_branches_api_url, timeout=60).json()
|
||||||
epel = False
|
epel = False
|
||||||
fedora = False
|
fedora = False
|
||||||
|
@ -562,7 +583,7 @@ if __name__ == '__main__':
|
||||||
while True:
|
while True:
|
||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
print('Querying {0}'.format(pagure_container_api_url))
|
print('Querying {0}'.format(pagure_container_api_url))
|
||||||
rv_json = requests.get(pagure_container_api_url, timeout=120).json()
|
rv_json = session.get(pagure_container_api_url, timeout=120).json()
|
||||||
for project in rv_json['projects']:
|
for project in rv_json['projects']:
|
||||||
project_pkgdb_schema = pagure_project_to_acl_schema(project)
|
project_pkgdb_schema = pagure_project_to_acl_schema(project)
|
||||||
projects_dict['Fedora Container'][project['name']] = \
|
projects_dict['Fedora Container'][project['name']] = \
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue