62405a2753
From the logs of this cron job running, it appears that every now and then
a request to pagure isn't returning JSON data. Could be because the server
got restarted in the middle or something else.
The result is however the same, the request's data can't be converted to
JSON.
This commit is an addition to 0db6035454 which
was already taking care of a similar issue but when retrieving project
information, this one is for retrieving groups information.
Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>
110 lines
3.2 KiB
Python
110 lines
3.2 KiB
Python
#!/usr/bin/python -tt
|
|
|
|
"""
|
|
This script is ran as a cronjob and bastion.
|
|
|
|
Its goal is to generate all the <pkg>-owner email aliases we provide
|
|
"""
|
|
|
|
import time
|
|
|
|
import requests
|
|
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.packages.urllib3.util.retry import Retry
|
|
|
|
|
|
def retry_session():
|
|
session = requests.Session()
|
|
retry = Retry(
|
|
total=5,
|
|
read=5,
|
|
connect=5,
|
|
backoff_factor=0.3,
|
|
status_forcelist=(500, 502, 504),
|
|
)
|
|
adapter = HTTPAdapter(max_retries=retry)
|
|
session.mount('http://', adapter)
|
|
session.mount('https://', adapter)
|
|
return session
|
|
|
|
|
|
pagure_url = 'https://src.fedoraproject.org/'
|
|
pagure_group_url = pagure_url + '/api/0/group/{group}'
|
|
project_to_email = {}
|
|
|
|
|
|
def get_pagure_projects():
|
|
pagure_projects_url = pagure_url + '/api/0/projects?page=1&per_page=100&fork=false'
|
|
session = retry_session()
|
|
while pagure_projects_url:
|
|
cnt = 0
|
|
while True:
|
|
try:
|
|
response = session.get(pagure_projects_url)
|
|
data = response.json()
|
|
break
|
|
except Exception:
|
|
if cnt == 4:
|
|
raise
|
|
|
|
cnt += 1
|
|
time.sleep(30)
|
|
|
|
for project in data['projects']:
|
|
yield project
|
|
# This is set to None on the last page.
|
|
pagure_projects_url = data['pagination']['next']
|
|
|
|
|
|
session = retry_session()
|
|
group_data = {}
|
|
for project in get_pagure_projects():
|
|
users = set(project['access_users']['owner']) | \
|
|
set(project['access_users']['admin']) | \
|
|
set(project['access_users']['commit'])
|
|
groups = set()
|
|
for group_kind in ('admin', 'commit'):
|
|
for group in project['access_groups'][group_kind]:
|
|
groups.add(group)
|
|
|
|
for group in groups:
|
|
if group in group_data:
|
|
users = users | group_data[group]
|
|
continue
|
|
|
|
cnt = 0
|
|
while True:
|
|
try:
|
|
response = session.get(pagure_group_url.format(group=group))
|
|
data = response.json()
|
|
break
|
|
except Exception:
|
|
if cnt == 4:
|
|
raise
|
|
cnt += 1
|
|
time.sleep(30)
|
|
|
|
group_members = data['members']
|
|
users = users | set(group_members)
|
|
group_data[group] = set(group_members)
|
|
|
|
project_alias = '{0}-owner'.format(project['name'])
|
|
# If there is a namespace, prefix the email with it plus a dash
|
|
if project['namespace'] and project['namespace'] != 'rpms':
|
|
project_alias = '{0}-{1}'.format(project['namespace'], project_alias)
|
|
|
|
# Use the @fedoraproject.org email alias instead of looking their email up
|
|
# in FAS
|
|
emails = ['{0}@fedoraproject.org'.format(user) for user in users]
|
|
|
|
# Handle case-insensitivity in postfix by unioning things.
|
|
project_alias = project_alias.lower()
|
|
if project_alias in project_to_email:
|
|
project_to_email[project_alias] = project_to_email[project_alias].union(emails)
|
|
else:
|
|
project_to_email[project_alias] = set(emails)
|
|
|
|
|
|
for project_alias, emails in project_to_email.items():
|
|
print('{0}: {1}'.format(project_alias, ','.join(sorted(emails))))
|