From e544f90a6ee922c1dd983b90443ddd078be310e4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 4 Aug 2017 20:36:39 +0000 Subject: [PATCH 1/7] Email alias script - use real pagure url. --- roles/packager_alias/files/owner-email-from-pagure.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/packager_alias/files/owner-email-from-pagure.py b/roles/packager_alias/files/owner-email-from-pagure.py index f783dfe7bf..df64ffe908 100644 --- a/roles/packager_alias/files/owner-email-from-pagure.py +++ b/roles/packager_alias/files/owner-email-from-pagure.py @@ -8,8 +8,7 @@ Its goal is to generate all the -owner email aliases we provide import requests -# TODO: Change me -pagure_url = 'http://127.0.0.1:5000' +pagure_url = 'https://src.fedoraproject.org/' pagure_group_url = pagure_url + '/api/0/group/{group}' pagure_projects_url = pagure_url + '/api/0/projects' pagure_projects = requests.get(pagure_projects_url).json()['projects'] From fbfb7c41f6a3d3932e0203b774458bcf6de5f5a2 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 4 Aug 2017 20:37:26 +0000 Subject: [PATCH 2/7] Email alias script - use pagination so as not to break pagure's golden heart. --- .../files/owner-email-from-pagure.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/roles/packager_alias/files/owner-email-from-pagure.py b/roles/packager_alias/files/owner-email-from-pagure.py index df64ffe908..43bce97037 100644 --- a/roles/packager_alias/files/owner-email-from-pagure.py +++ b/roles/packager_alias/files/owner-email-from-pagure.py @@ -10,11 +10,21 @@ import requests pagure_url = 'https://src.fedoraproject.org/' pagure_group_url = pagure_url + '/api/0/group/{group}' -pagure_projects_url = pagure_url + '/api/0/projects' -pagure_projects = requests.get(pagure_projects_url).json()['projects'] project_to_email = {} -for project in pagure_projects: + +def get_pagure_projects(): + pagure_projects_url = pagure_url + '/api/0/projects?page=1&per_page=100' + while pagure_projects_url: + response = requests.get(pagure_projects_url) + data = response.json() + for project in data['projects']: + yield project + # This is set to None on the last page. + pagure_projects_url = data['pagination']['next'] + + +for project in get_pagure_projects(): users = set(project['access_users']['owner']) | \ set(project['access_users']['admin']) | \ set(project['access_users']['commit']) From eb241f903dcd6d7d3529539739508f6dc1b00982 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 4 Aug 2017 20:39:05 +0000 Subject: [PATCH 3/7] Flip over packager email alias generation to use the new script. --- roles/packager_alias/tasks/main.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/roles/packager_alias/tasks/main.yml b/roles/packager_alias/tasks/main.yml index 050e4c0a9c..8a6ae1f659 100644 --- a/roles/packager_alias/tasks/main.yml +++ b/roles/packager_alias/tasks/main.yml @@ -1,24 +1,12 @@ --- # Email alias set-up -- name: Install the Python script to get the -owner email alias (non-staging) - copy: - src: owner-email.py - dest: /usr/local/bin/owner-email.py - owner: root - group: root - mode: 0755 - when: env != 'staging' - tags: - - install - -- name: Install the Python script to get the -owner email alias (staging) +- name: Install the Python script to get the -owner email alias copy: src: owner-email-from-pagure.py dest: /usr/local/bin/owner-email.py owner: root group: root mode: 0755 - when: env == 'staging' tags: - install From 9653f40afde748038a34132f5ef86f19e2e6d1f1 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 4 Aug 2017 20:39:29 +0000 Subject: [PATCH 4/7] Remove the old packager email alias generation script. --- roles/packager_alias/files/owner-email.py | 79 ----------------------- 1 file changed, 79 deletions(-) delete mode 100755 roles/packager_alias/files/owner-email.py diff --git a/roles/packager_alias/files/owner-email.py b/roles/packager_alias/files/owner-email.py deleted file mode 100755 index d4099b7497..0000000000 --- a/roles/packager_alias/files/owner-email.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/python -tt - -""" -This script is ran as a cronjob and bastion. - -Its goal is to generate all the -owner email aliases we provide -""" - -import os -import sys -import urllib2 -from fedora.client import BaseClient, ServerError, AuthError -import ConfigParser -import requests - -config = ConfigParser.ConfigParser() -config.read('/etc/fas.conf') - - -pkgdb_url = 'https://admin.fedoraproject.org/pkgdb' -fas = BaseClient('https://admin.fedoraproject.org/accounts', - username=config.get('global', 'login').strip('"'), - password=config.get('global', 'password').strip('"')) - -try: - pkgdb_data = requests.get('%s/api/notify/?format=json' % pkgdb_url, - verify=True).json() - fas_data = fas.send_request('/user/email_list', auth=True) - fas_groups = fas.send_request( - '/group/type_list', auth=True, req_params={'grptype': 'pkgdb'}) -except ServerError, e: - print >> sys.stderr, '%s' % e - sys.exit(1) -except AuthError, e: - print >> sys.stderr, '%s: %s' % (e.exc, e.message) - sys.exit(1) -except (urllib2.HTTPError,urllib2.URLError), e: - print >> sys.stderr, '%s' % e - sys.exit(1) - -else: - pkgs = pkgdb_data['packages'] - if len(pkgs) < 500: - print >> sys.stderr, 'Too few packages, something is wrong' - sys.exit(1) - email_list = fas_data['emails'] - group_mail = {} - for group in fas_groups.groups: - group_mail[group.name] = group.mailing_list - - contactlist = {} - for pkg, ccusers in pkgs.iteritems(): - emails = [] - for user in ccusers: - if user in email_list: - emails.append(email_list[user]) - elif user.startswith('group::'): - user = user.replace('group::', '') - if user in group_mail and group_mail[user]: - emails.append(group_mail[user]) - else: - print >> sys.stderr, 'Strange user `%s`, not in '\ - 'email_list nor in group_mail, badly set-up '\ - 'group?\n' % user - else: - print >> sys.stderr, 'Strange user `%s`, not in '\ - 'email_list and not a group\n' % user - - if pkg.lower() in contactlist: - contactlist[pkg.lower()] = contactlist[pkg.lower()].union(emails) - else: - contactlist[pkg.lower()] = set(emails) - - for pkg, emails in sorted(contactlist.iteritems()): - print '%s-owner: %s' % (pkg.lower(),','.join(sorted(emails))) - - for group in fas_groups.groups: - print '{0}: {1}'.format(group.name, group.mailing_list) - From e4a44181f7fe9783454c002c70b0d6a7bba45f5c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 4 Aug 2017 20:41:10 +0000 Subject: [PATCH 5/7] Tag this role with its own name. --- roles/packager_alias/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/packager_alias/tasks/main.yml b/roles/packager_alias/tasks/main.yml index 8a6ae1f659..b22c4892e9 100644 --- a/roles/packager_alias/tasks/main.yml +++ b/roles/packager_alias/tasks/main.yml @@ -9,6 +9,7 @@ mode: 0755 tags: - install + - packager_alias - name: Install the script to generate the -owner email alias copy: @@ -19,6 +20,7 @@ mode: 0755 tags: - install + - packager_alias # # Since this host has mail aliases, it's a mail hub. Compress logs since there will be a ton of them @@ -28,3 +30,4 @@ copy: src=syslog dest=/etc/logrotate.d/syslog owner=root group=root tags: - install + - packager_alias From 4c93ad1d0ac00b3ca08edf7a0cdfbc9c6814cb2a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 4 Aug 2017 22:46:28 +0200 Subject: [PATCH 6/7] Adjust the pdc url in stg vs prod --- roles/distgit/pagure/templates/pagure.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/distgit/pagure/templates/pagure.cfg b/roles/distgit/pagure/templates/pagure.cfg index 44f294055d..8973c0a674 100644 --- a/roles/distgit/pagure/templates/pagure.cfg +++ b/roles/distgit/pagure/templates/pagure.cfg @@ -210,7 +210,11 @@ REQUIRED_GROUPS = { 'rpms/*': ['packager'], } +{% if env == 'staging' %} PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/' +{% else %} +PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1/' +{% endif %} GITOLITE_BACKEND = 'distgit' From 3fb63225412adbf3d35502b93162ccb3f800588b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 4 Aug 2017 23:01:34 +0200 Subject: [PATCH 7/7] Actually we need python-fedmsg-genacls and its config file --- roles/distgit/tasks/main.yml | 14 +++++++------- roles/distgit/templates/fedmsg-genacls-config.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index bf9eaafe3c..80ac4571bf 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -12,6 +12,7 @@ - httpd - mod_ssl - mod_auth_gssapi + - python-fedmsg-genacls - /usr/sbin/semanage tags: - distgit @@ -219,6 +220,12 @@ - config - distgit +- name: install the fedmsg configuration + template: src=fedmsg-genacls-config.py dest=/etc/fedmsg.d/genacls.py + owner=root group=root mode=0644 + tags: + - distgit + - name: remove file pre-dating pagure over dist-git file: path={{ item }} state=absent with_items: @@ -230,13 +237,6 @@ - config - distgit -- name: remove packages pre-dating pagure over dist-git - yum: pkg={{ item }} state=absent - with_items: - - python-fedmsg-genacls - tags: - - distgit - - name: Get admin users command: "/srv/web/infra/ansible/scripts/users-from-fas @sysadmin-main {{ admin_groups }}" register: admin_user_list diff --git a/roles/distgit/templates/fedmsg-genacls-config.py b/roles/distgit/templates/fedmsg-genacls-config.py index 58d3acaede..419dcbd01c 100644 --- a/roles/distgit/templates/fedmsg-genacls-config.py +++ b/roles/distgit/templates/fedmsg-genacls-config.py @@ -1,5 +1,5 @@ config = { - 'genacls.consumer.enabled': True, + 'genacls.consumer.enabled': False, 'genacls.consumer.delay': 5, # 5 seconds # New world