From 8cd38d6a4a3fe199390263c5dc6050aa935f5c15 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 9 Aug 2017 11:25:53 +0200 Subject: [PATCH 01/17] First attempt to get multi-static work --- roles/distgit/pagure/templates/z_pagure.conf | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/roles/distgit/pagure/templates/z_pagure.conf b/roles/distgit/pagure/templates/z_pagure.conf index 4ccdf28b8e..408bbc6351 100644 --- a/roles/distgit/pagure/templates/z_pagure.conf +++ b/roles/distgit/pagure/templates/z_pagure.conf @@ -44,3 +44,12 @@ WSGIDaemonProcess pagureproc user=pagure group=packager maximum-requests=1000 di # + +RewriteEngine on + +# First try the instance-specific theme +RewriteCond "/usr/share/pagure_dist_git/static/$1" -f +RewriteRule "^/static/(.*)" "/usr/share/pagure_dist_git/static/$1" [L] + +# Use the application default theme for files not customized +RewriteRule "^/static/(.*)" "/usr/lib/python2.7/site-packages/pagure/static/$1" [L] From ae901a21bb66e3c4e35f1c8a1e242b7afe9ff55c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 9 Aug 2017 11:52:36 +0200 Subject: [PATCH 02/17] Blacklist 'group' as group name --- roles/distgit/pagure/templates/pagure.cfg | 1 + roles/pagure/frontend/templates/pagure.cfg | 2 ++ 2 files changed, 3 insertions(+) diff --git a/roles/distgit/pagure/templates/pagure.cfg b/roles/distgit/pagure/templates/pagure.cfg index 7f4d6b98ba..ab5ca71493 100644 --- a/roles/distgit/pagure/templates/pagure.cfg +++ b/roles/distgit/pagure/templates/pagure.cfg @@ -246,3 +246,4 @@ ADMIN_API_ACLS = [ 'create_project', ] +BLACKLISTED_GROUPS = ['forks', 'group'] diff --git a/roles/pagure/frontend/templates/pagure.cfg b/roles/pagure/frontend/templates/pagure.cfg index 777de4078a..317c8d43c4 100644 --- a/roles/pagure/frontend/templates/pagure.cfg +++ b/roles/pagure/frontend/templates/pagure.cfg @@ -282,3 +282,5 @@ CROSS_PROJECT_ACLS = [ 'issue_create', 'issue_comment', ] + +BLACKLISTED_GROUPS = ['forks', 'group'] From dcd69e0036b45252d11972c3f559f2848b024d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 9 Aug 2017 10:49:12 +0000 Subject: [PATCH 03/17] HyperKitty: store more async results for stats --- roles/mailman/templates/settings.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 index ba896da70b..d088f08b51 100644 --- a/roles/mailman/templates/settings.py.j2 +++ b/roles/mailman/templates/settings.py.j2 @@ -323,7 +323,7 @@ HAYSTACK_CONNECTIONS = { # Q_CLUSTER = { 'timeout': 300, - 'save_limit': 500, + 'save_limit': 100000, 'orm': 'default', } From 9a933e3bb52d391c21ffcce3cb7e9679af115bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 9 Aug 2017 13:21:22 +0200 Subject: [PATCH 04/17] make 'restart fedmsg-hub' handler not exclusive to bodhi --- handlers/restart_services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/restart_services.yml b/handlers/restart_services.yml index f5f3b4d209..eecf439bb6 100644 --- a/handlers/restart_services.yml +++ b/handlers/restart_services.yml @@ -22,7 +22,7 @@ # Note that, we're cool with arbitrary restarts on bodhi-backend02, just # not bodhi-backend01 or bodhi-backend03. 01 and 03 is where the releng/mash # stuff happens and we # don't want to interrupt that. - when: inventory_hostname == 'bodhi-backend02.phx2.fedoraproject.org' + when: inventory_hostname not in ['bodhi-backend01.phx2.fedoraproject.org', 'bodhi-backend03.phx2.fedoraproject.org'] - name: restart fedmsg-irc command: /usr/local/bin/conditional-restart.sh fedmsg-irc fedmsg-irc From bcc4d0bba6232a0722b5c6a86d1a172486769ec1 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Tue, 18 Jul 2017 14:48:39 +0000 Subject: [PATCH 05/17] Add the unmodified version of pkgdb-sync-bugzilla.py from GitHub in preparation for modification --- .../templates/pagure-sync-bugzilla.py.j2 | 471 ++++++++++++++++++ 1 file changed, 471 insertions(+) create mode 100644 roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 new file mode 100644 index 0000000000..166163d37b --- /dev/null +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -0,0 +1,471 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# +# Copyright © 2013-2016 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, modify, +# copy, or redistribute it subject to the terms and conditions of the GNU +# General Public License v.2, or (at your option) any later version. This +# program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the GNU +# General Public License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the source +# code or documentation are not subject to the GNU General Public License and +# may only be used or replicated with the express permission of Red Hat, Inc. +# +# Red Hat Author(s): Toshio Kuratomi +# Author(s): Mike Watters +# Author(s): Pierre-Yves Chibon +# +''' +sync information from the packagedb into bugzilla + +This short script takes information about package onwership and imports it +into bugzilla. +''' + +## These two lines are needed to run on EL6 +__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] +import pkg_resources + +import argparse +import datetime +import time +import sys +import os +import itertools +import json +import xmlrpclib +import codecs +import smtplib +import bugzilla +import requests +from email.Message import Message +from fedora.client.fas2 import AccountSystem + + +if 'PKGDB2_CONFIG' not in os.environ \ + and os.path.exists('/etc/pkgdb2/pkgdb2.cfg'): + print 'Using configuration file `/etc/pkgdb2/pkgdb2.cfg`' + os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg' + + +try: + import pkgdb2 +except ImportError: + sys.path.insert( + 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) + import pkgdb2 + + +BZSERVER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_URL') +BZUSER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_USER') +BZPASS = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_PASSWORD') +BZCOMPAPI = pkgdb2.APP.config.get('BUGZILLA_COMPONENT_API') +FASURL = pkgdb2.APP.config.get('PKGDB2_FAS_URL') +FASUSER = pkgdb2.APP.config.get('PKGDB2_FAS_USER') +FASPASS = pkgdb2.APP.config.get('PKGDB2_FAS_PASSWORD') +FASINSECURE = pkgdb2.APP.config.get('PKGDB2_FAS_INSECURE') +NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL') +PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL') +DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False) + +EMAIL_FROM = 'accounts@fedoraproject.org' +DATA_CACHE = '/var/tmp/pkgdb_sync_bz.json' + +PRODUCTS = { + 'Fedora': 'Fedora', + 'Fedora Docker': 'Fedora Docker Images', + 'Fedora Container': 'Fedora Container Images', + 'Fedora EPEL': 'Fedora EPEL', +} + +PRODUCTS = pkgdb2.APP.config.get('BZ_PRODUCTS', PRODUCTS) + +# When querying for current info, take segments of 1000 packages a time +BZ_PKG_SEGMENT = 1000 + + +TMPL_EMAIL_ADMIN = ''' +The following errors were encountered while updating bugzilla with information +from the Package Database. Please have the problems taken care of: + +%s +''' + + +class DataChangedError(Exception): + '''Raised when data we are manipulating changes while we're modifying it.''' + pass + + +def segment(iterable, chunk, fill=None): + '''Collect data into `chunk` sized block''' + args = [iter(iterable)] * chunk + return itertools.izip_longest(*args, fillvalue=fill) + + +class ProductCache(dict): + def __init__(self, bz, acls): + self.bz = bz + self.acls = acls + + # Ask bugzilla for a section of the pkglist. + # Save the information from the section that we want. + def __getitem__(self, key): + try: + return super(ProductCache, self).__getitem__(key) + except KeyError: + # We can only cache products we have pkgdb information for + if key not in self.acls: + raise + + if BZCOMPAPI == 'getcomponentsdetails': + # Old API -- in python-bugzilla. But with current server, this + # gives ProxyError + products = self.server.getcomponentsdetails(key) + elif BZCOMPAPI == 'component.get': + # Way that's undocumented in the partner-bugzilla api but works + # currently + pkglist = acls[key].keys() + products = {} + for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): + # Format that bugzilla will understand. Strip None's that segment() pads + # out the final data segment() with + query = [ + dict(product=PRODUCTS[key], component=p) + for p in pkg_segment if p is not None + ] + raw_data = self.bz._proxy.Component.get(dict(names=query)) + for package in raw_data['components']: + # Reformat data to be the same as what's returned from + # getcomponentsdetails + product = dict(initialowner=package['default_assignee'], + description=package['description'], + initialqacontact=package['default_qa_contact'], + initialcclist=package['default_cc']) + products[package['name'].lower()] = product + self[key] = products + + return super(ProductCache, self).__getitem__(key) + + +class Bugzilla(object): + + def __init__(self, bzServer, username, password, acls): + self.bzXmlRpcServer = bzServer + self.username = username + self.password = password + + self.server = bugzilla.Bugzilla( + url=self.bzXmlRpcServer, + user=self.username, + password=self.password) + self.productCache = ProductCache(self.server, acls) + + # Connect to the fedora account system + self.fas = AccountSystem( + base_url=FASURL, + username=FASUSER, + password=FASPASS) + self.userCache = self.fas.people_by_key( + key='username', + fields=['bugzilla_email']) + + def _get_bugzilla_email(self, username): + '''Return the bugzilla email address for a user. + + First looks in a cache for a username => bugzilla email. If not found, + reloads the cache from fas and tries again. + ''' + try: + return self.userCache[username]['bugzilla_email'].lower() + except KeyError: + if username.startswith('@'): + group = self.fas.group_by_name(username[1:]) + self.userCache[username] = { + 'bugzilla_email': group.mailing_list} + else: + person = self.fas.person_by_username(username) + bz_email = person.get('bugzilla_email', None) + if bz_email is None: + print '%s has no bugzilla email, valid account?' % username + else: + self.userCache[username] = {'bugzilla_email': bz_email} + return self.userCache[username]['bugzilla_email'].lower() + + def add_edit_component(self, package, collection, owner, description, + qacontact=None, cclist=None): + '''Add or update a component to have the values specified. + ''' + # Turn the cclist into something usable by bugzilla + if not cclist or 'people' not in cclist: + initialCCList = list() + else: + initialCCList = [ + self._get_bugzilla_email(cc) for cc in cclist['people']] + if 'groups' in cclist: + group_cc = [ + self._get_bugzilla_email(cc) for cc in cclist['groups']] + initialCCList.extend(group_cc) + + # Add owner to the cclist so comaintainers taking over a bug don't + # have to do this manually + owner = self._get_bugzilla_email(owner) + if owner not in initialCCList: + initialCCList.append(owner) + + # Lookup product + try: + product = self.productCache[collection] + except xmlrpclib.Fault as e: + # Output something useful in args + e.args = (e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError as e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + + pkgKey = package.lower() + if pkgKey in product: + # edit the package information + data = {} + + # Grab bugzilla email for things changable via xmlrpc + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + # Check for changes to the owner, qacontact, or description + if product[pkgKey]['initialowner'] != owner: + data['initialowner'] = owner + + if product[pkgKey]['description'] != description: + data['description'] = description + if product[pkgKey]['initialqacontact'] != qacontact and ( + qacontact or product[pkgKey]['initialqacontact']): + data['initialqacontact'] = qacontact + + if len(product[pkgKey]['initialcclist']) != len(initialCCList): + data['initialcclist'] = initialCCList + else: + for ccMember in product[pkgKey]['initialcclist']: + if ccMember not in initialCCList: + data['initialcclist'] = initialCCList + break + + if data: + ### FIXME: initialowner has been made mandatory for some + # reason. Asking dkl why. + data['initialowner'] = owner + + # Changes occurred. Submit a request to change via xmlrpc + data['product'] = PRODUCTS[collection] + data['component'] = package + if DRY_RUN: + print '[EDITCOMP] Changing via editComponent(' \ + '%s, %s, "xxxxx")' % (data, self.username) + print '[EDITCOMP] Former values: %s|%s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact'], + product[pkgKey]['initialcclist']) + else: + try: + self.server.editcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + except xmlrpclib.ProtocolError, e: + e.args = ('ProtocolError', e.errcode, e.errmsg) + raise + else: + # Add component + if qacontact: + qacontact = self._get_bugzilla_email(qacontact) + else: + qacontact = 'extras-qa@fedoraproject.org' + + data = { + 'product': PRODUCTS[collection], + 'component': package, + 'description': description, + 'initialowner': owner, + 'initialqacontact': qacontact + } + if initialCCList: + data['initialcclist'] = initialCCList + + if DRY_RUN: + print '[ADDCOMP] Adding new component AddComponent:(' \ + '%s, %s, "xxxxx")' % (data, self.username) + else: + try: + self.server.addcomponent(data) + except xmlrpclib.Fault, e: + # Output something useful in args + e.args = (data, e.faultCode, e.faultString) + raise + + +def send_email(fromAddress, toAddress, subject, message, ccAddress=None): + '''Send an email if there's an error. + + This will be replaced by sending messages to a log later. + ''' + msg = Message() + msg.add_header('To', ','.join(toAddress)) + msg.add_header('From', fromAddress) + msg.add_header('Subject', subject) + if ccAddress is not None: + msg.add_header('Cc', ','.join(ccAddress)) + toAddress = toAddress + ccAddress + msg.set_payload(message) + smtp = smtplib.SMTP('bastion') + smtp.sendmail(fromAddress, toAddress, msg.as_string()) + smtp.quit() + + +def notify_users(errors): + ''' Browse the list of errors and when we can retrieve the email + address, use it to notify the user about the issue. + ''' + tmpl_email = pkgdb2.APP.config.get('PKGDB_SYNC_BUGZILLA_EMAIL', None) + if not tmpl_email: + print 'No template email configured in the configuration file, '\ + 'no notification sent to the users' + return + + data = {} + if os.path.exists(DATA_CACHE): + try: + with open(DATA_CACHE) as stream: + data = json.load(stream) + except Exception as err: + print 'Could not read the json file at %s: \nError: %s' % ( + DATA_CACHE, err) + + new_data = {} + seen = [] + for error in errors: + notify_user = False + if 'The name ' in error and ' is not a valid username' in error: + user_email = error.split(' is not a valid username')[0].split( + 'The name ')[1].strip() + now = datetime.datetime.utcnow() + + # See if we already know about this user + if user_email in data and data[user_email]['last_update']: + last_update = datetime.datetime.fromtimestamp( + int(data[user_email]['last_update'])) + # Only notify users once per hour + if (now - last_update).seconds >= 3600: + notify_user = True + else: + new_data[user_email] = data[user_email] + elif not data or user_email not in data: + notify_user = True + + # Ensure we notify the user only once, no matter how many errors we + # got concerning them. + if user_email not in seen: + seen.append(user_email) + else: + notify_user = False + + if notify_user: + send_email( + EMAIL_FROM, + [user_email], + subject='Please fix your bugzilla.redhat.com account', + message=tmpl_email, + ccAddress=NOTIFYEMAIL, + ) + + new_data[user_email] = { + 'last_update': time.mktime(now.timetuple()) + } + + with open(DATA_CACHE, 'w') as stream: + json.dump(new_data, stream) + + +if __name__ == '__main__': + sys.stdout = codecs.getwriter('utf-8')(sys.stdout) + + + parser = argparse.ArgumentParser( + description='Script syncing information between pkgdb and bugzilla' + ) + parser.add_argument( + '--debug', dest='debug', action='store_true', default=False, + help='Print the changes instead of making them in bugzilla') + + args = parser.parse_args() + + if args.debug: + DRY_RUN = True + + # Non-fatal errors to alert people about + errors = [] + + # Get bugzilla information from the package database + req = requests.get('%s/api/bugzilla/?format=json' % PKGDBSERVER) + acls = req.json()['bugzillaAcls'] + + # Initialize the connection to bugzilla + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls) + + for product in acls.keys(): + if product not in PRODUCTS: + continue + for pkg in sorted(acls[product]): + if DRY_RUN: + print pkg + pkgInfo = acls[product][pkg] + try: + bugzilla.add_edit_component( + pkg, + product, + pkgInfo['owner'], + pkgInfo['summary'], + pkgInfo['qacontact'], + pkgInfo['cclist']) + except ValueError, e: + # A username didn't have a bugzilla address + errors.append(str(e.args)) + except DataChangedError, e: + # A Package or Collection was returned via xmlrpc but wasn't + # present when we tried to change it + errors.append(str(e.args)) + except xmlrpclib.ProtocolError, e: + # Unrecoverable and likely means that nothing is going to + # succeed. + errors.append(str(e.args)) + break + except xmlrpclib.Error, e: + # An error occurred in the xmlrpc call. Shouldn't happen but + # we better see what it is + errors.append('%s -- %s' % (pkg, e.args[-1])) + + # Send notification of errors + if errors: + if DRY_RUN: + print '[DEBUG]', '\n'.join(errors) + else: + notify_users(errors) + send_email( + EMAIL_FROM, + NOTIFYEMAIL, + 'Errors while syncing bugzilla with the PackageDB', + TMPL_EMAIL_ADMIN % ('\n'.join(errors),)) + else: + with open(DATA_CACHE, 'w') as stream: + json.dump({}, stream) + + sys.exit(0) From c842d232d2da662dc64a27f9d8dba988e03ed0a7 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Thu, 20 Jul 2017 20:25:17 +0000 Subject: [PATCH 06/17] Convert the pkgdb-sync-bugzilla.py script to pagure-sync-bugzilla.py and run it on Pagure over dist-git This is part of https://fedoraproject.org/wiki/Changes/ArbitraryBranching. Since PkgDB will be decommissioned, we need to start using Pagure's API instead of PkgDB to sync Bugzilla component owners and CC users. --- roles/distgit/pagure/tasks/main.yml | 26 ++ .../templates/pagure-sync-bugzilla.py.j2 | 296 +++++++++++++----- 2 files changed, 240 insertions(+), 82 deletions(-) diff --git a/roles/distgit/pagure/tasks/main.yml b/roles/distgit/pagure/tasks/main.yml index 88e20b9ab4..3df3ba5d08 100644 --- a/roles/distgit/pagure/tasks/main.yml +++ b/roles/distgit/pagure/tasks/main.yml @@ -14,6 +14,9 @@ - libsemanage-python - python-fedora-flask - python2-pagure-dist-git + # For the pagure-sync-bugzilla.py script + - python-bugzilla + - python2-requests # - mod_ssl # - stunnel tags: @@ -171,6 +174,29 @@ - pagure +- name: generate pagure-sync-bugzilla.py script + template: + src: pagure-sync-bugzilla.py.j2 + dest: /usr/local/bin/pagure-sync-bugzilla.py + owner: root + group: root + mode: 0700 + tags: + - pagure + + +- name: Configure cron job for a daily pagure-sync-bugzilla.py script run + cron: + name: pagure-sync-bugzilla + user: root + minute: 0 + hour: 18 + job: /usr/local/bin/pagure-sync-bugzilla + cron_file: pagure-sync-bugzilla + state: present + tags: + - pagure + # Ensure all the services are up and running - name: Start and enable httpd, postfix, pagure_milter diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 166163d37b..ceb6b86314 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -21,16 +21,13 @@ # Author(s): Pierre-Yves Chibon # ''' -sync information from the packagedb into bugzilla +sync information from the Pagure into bugzilla This short script takes information about package onwership and imports it into bugzilla. ''' -## These two lines are needed to run on EL6 -__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] -import pkg_resources - +import re import argparse import datetime import time @@ -43,48 +40,45 @@ import codecs import smtplib import bugzilla import requests -from email.Message import Message +try: + from email.Message import Message +except ImportError: + from email.message import EmailMessage as Message from fedora.client.fas2 import AccountSystem +BZSERVER = 'https://bugzilla.redhat.com' +BZUSER = '{{ bugzilla_user }}' +BZPASS = '{{ bugzilla_password }}' +BZCOMPAPI = 'component.get' +FASUSER = '{{ fedorathirdpartyUser }}' +FASPASS = '{{ fedorathirdpartyPassword }}' +NOTIFYEMAIL = [ + 'kevin@fedoraproject.org', + 'pingou@fedoraproject.org'] +DRY_RUN = False -if 'PKGDB2_CONFIG' not in os.environ \ - and os.path.exists('/etc/pkgdb2/pkgdb2.cfg'): - print 'Using configuration file `/etc/pkgdb2/pkgdb2.cfg`' - os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg' +{% if env == 'staging' %} +FASURL = 'https://admin.stg.fedoraproject.org/accounts' +FASINSECURE = True +PAGUREURL = 'https://src.stg.fedoraproject.org/pagure/' +MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/' +{% else %} +FASURL = 'https://admin.fedoraproject.org/accounts' +FASINSECURE = False +PAGUREURL = 'https://src.fedoraproject.org/pagure/' +MDAPIURL = 'https://apps.fedoraproject.org/mdapi/' +{% endif %} -try: - import pkgdb2 -except ImportError: - sys.path.insert( - 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) - import pkgdb2 - - -BZSERVER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_URL') -BZUSER = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_USER') -BZPASS = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_PASSWORD') -BZCOMPAPI = pkgdb2.APP.config.get('BUGZILLA_COMPONENT_API') -FASURL = pkgdb2.APP.config.get('PKGDB2_FAS_URL') -FASUSER = pkgdb2.APP.config.get('PKGDB2_FAS_USER') -FASPASS = pkgdb2.APP.config.get('PKGDB2_FAS_PASSWORD') -FASINSECURE = pkgdb2.APP.config.get('PKGDB2_FAS_INSECURE') -NOTIFYEMAIL = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_NOTIFY_EMAIL') -PKGDBSERVER = pkgdb2.APP.config.get('SITE_URL') -DRY_RUN = pkgdb2.APP.config.get('PKGDB2_BUGZILLA_DRY_RUN', False) - EMAIL_FROM = 'accounts@fedoraproject.org' -DATA_CACHE = '/var/tmp/pkgdb_sync_bz.json' +DATA_CACHE = '/var/tmp/pagure_sync_bz.json' PRODUCTS = { 'Fedora': 'Fedora', - 'Fedora Docker': 'Fedora Docker Images', 'Fedora Container': 'Fedora Container Images', 'Fedora EPEL': 'Fedora EPEL', } -PRODUCTS = pkgdb2.APP.config.get('BZ_PRODUCTS', PRODUCTS) - # When querying for current info, take segments of 1000 packages a time BZ_PKG_SEGMENT = 1000 @@ -96,6 +90,41 @@ from the Package Database. Please have the problems taken care of: %s ''' +# PkgDB sync bugzilla email +PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings. + +You are receiving this email because there's a problem with your +bugzilla.redhat.com account. + +If you recently changed the email address associated with your +Fedora account in the Fedora Account System, it is now out of sync +with your bugzilla.redhat.com account. This leads to problems +with Fedora packages you own or are CC'ed on bug reports for. + +Please take one of the following actions: + +a) login to your old bugzilla.redhat.com account and change the email +address to match your current email in the Fedora account system. +https://bugzilla.redhat.com login, click preferences, account +information and enter new email address. + +b) Create a new account in bugzilla.redhat.com to match your +email listed in your Fedora account system account. +https://bugzilla.redhat.com/ click 'new account' and enter email +address. + +c) Change your Fedora Account System email to match your existing +bugzilla.redhat.com account. +https://admin.fedoraproject.org/accounts login, click on 'my account', +then 'edit' and change your email address. + +If you have questions or concerns, please let us know. + +Your prompt attention in this matter is appreciated. + +The Fedora admins. +""" + class DataChangedError(Exception): '''Raised when data we are manipulating changes while we're modifying it.''' @@ -119,7 +148,7 @@ class ProductCache(dict): try: return super(ProductCache, self).__getitem__(key) except KeyError: - # We can only cache products we have pkgdb information for + # We can only cache products we have pagure information for if key not in self.acls: raise @@ -130,7 +159,7 @@ class ProductCache(dict): elif BZCOMPAPI == 'component.get': # Way that's undocumented in the partner-bugzilla api but works # currently - pkglist = acls[key].keys() + pkglist = projects_dict[key].keys() products = {} for pkg_segment in segment(pkglist, BZ_PKG_SEGMENT): # Format that bugzilla will understand. Strip None's that segment() pads @@ -192,12 +221,13 @@ class Bugzilla(object): person = self.fas.person_by_username(username) bz_email = person.get('bugzilla_email', None) if bz_email is None: - print '%s has no bugzilla email, valid account?' % username + print('%s has no bugzilla email, valid account?' + % username) else: self.userCache[username] = {'bugzilla_email': bz_email} return self.userCache[username]['bugzilla_email'].lower() - def add_edit_component(self, package, collection, owner, description, + def add_edit_component(self, package, collection, owner, description=None, qacontact=None, cclist=None): '''Add or update a component to have the values specified. ''' @@ -244,7 +274,7 @@ class Bugzilla(object): if product[pkgKey]['initialowner'] != owner: data['initialowner'] = owner - if product[pkgKey]['description'] != description: + if description and product[pkgKey]['description'] != description: data['description'] = description if product[pkgKey]['initialqacontact'] != qacontact and ( qacontact or product[pkgKey]['initialqacontact']): @@ -267,21 +297,21 @@ class Bugzilla(object): data['product'] = PRODUCTS[collection] data['component'] = package if DRY_RUN: - print '[EDITCOMP] Changing via editComponent(' \ - '%s, %s, "xxxxx")' % (data, self.username) - print '[EDITCOMP] Former values: %s|%s|%s|%s' % ( - product[pkgKey]['initialowner'], - product[pkgKey]['description'], - product[pkgKey]['initialqacontact'], - product[pkgKey]['initialcclist']) + print('[EDITCOMP] Changing via editComponent(' + '%s, %s, "xxxxx")' % (data, self.username)) + print('[EDITCOMP] Former values: %s|%s|%s|%s' % ( + product[pkgKey]['initialowner'], + product[pkgKey]['description'], + product[pkgKey]['initialqacontact'], + product[pkgKey]['initialcclist'])) else: try: self.server.editcomponent(data) - except xmlrpclib.Fault, e: + except xmlrpclib.Fault as e: # Output something useful in args e.args = (data, e.faultCode, e.faultString) raise - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: e.args = ('ProtocolError', e.errcode, e.errmsg) raise else: @@ -294,7 +324,7 @@ class Bugzilla(object): data = { 'product': PRODUCTS[collection], 'component': package, - 'description': description, + 'description': description or 'NA', 'initialowner': owner, 'initialqacontact': qacontact } @@ -302,12 +332,12 @@ class Bugzilla(object): data['initialcclist'] = initialCCList if DRY_RUN: - print '[ADDCOMP] Adding new component AddComponent:(' \ - '%s, %s, "xxxxx")' % (data, self.username) + print('[ADDCOMP] Adding new component AddComponent:(' + '%s, %s, "xxxxx")' % (data, self.username)) else: try: self.server.addcomponent(data) - except xmlrpclib.Fault, e: + except xmlrpclib.Fault as e: # Output something useful in args e.args = (data, e.faultCode, e.faultString) raise @@ -335,20 +365,14 @@ def notify_users(errors): ''' Browse the list of errors and when we can retrieve the email address, use it to notify the user about the issue. ''' - tmpl_email = pkgdb2.APP.config.get('PKGDB_SYNC_BUGZILLA_EMAIL', None) - if not tmpl_email: - print 'No template email configured in the configuration file, '\ - 'no notification sent to the users' - return - data = {} if os.path.exists(DATA_CACHE): try: with open(DATA_CACHE) as stream: data = json.load(stream) except Exception as err: - print 'Could not read the json file at %s: \nError: %s' % ( - DATA_CACHE, err) + print('Could not read the json file at %s: \nError: %s' % ( + DATA_CACHE, err)) new_data = {} seen = [] @@ -383,7 +407,7 @@ def notify_users(errors): EMAIL_FROM, [user_email], subject='Please fix your bugzilla.redhat.com account', - message=tmpl_email, + message=PKGDB_SYNC_BUGZILLA_EMAIL, ccAddress=NOTIFYEMAIL, ) @@ -395,12 +419,68 @@ def notify_users(errors): json.dump(new_data, stream) +def pagure_project_to_acl_schema(pagure_project): + """ + This function translates the JSON of a Pagure project to what PkgDB used to + output in the Bugzilla API. + :param pagure_project: a dictionary of the JSON of a Pagure project + :return: a dictionary of the content that the Bugzilla API would output + """ + base_error_msg = ('The connection to "{0}" failed with the status code ' + '{1} and output "{2}"') + watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format( + PAGUREURL.rstrip('/'), pagure_project['namespace'], + pagure_project['name']) + if DRY_RUN: + print('Querying {0}'.format(watchers_api_url)) + watchers_rv = requests.get(watchers_api_url, timeout=60) + if not watchers_rv.ok: + error_msg = base_error_msg.format( + watchers_api_url, watchers_rv.status_code, watchers_rv.text) + raise RuntimeError(error_msg) + watchers_rv_json = watchers_rv.json() + + user_cc_list = [] + for user, watch_levels in watchers_rv_json['watchers'].items(): + # Only people watching commits should be CC'd + if 'commit' in watch_levels: + user_cc_list.append(user) + + summary = None + if pagure_project['namespace'] != 'rpms': + mdapi_url = '{0}/rawhide/srcpkg/{1}'.format( + MDAPIURL.rstrip('/'), pagure_project['name']) + if DRY_RUN: + print('Querying {0}'.format(mdapi_url)) + mdapi_rv = requests.get(mdapi_url, timeout=60) + if mdapi_rv.ok: + mdapi_rv_json = mdapi_rv.json() + summary = mdapi_rv_json['summary'] + elif not mdapi_rv.ok and mdapi_rv.status_code != 404: + error_msg = base_error_msg.format( + mdapi_url, mdapi_rv.status_code, mdapi_rv.text) + raise RuntimeError(error_msg) + + return { + 'cclist': { + # Groups is empty because you can't have groups watch projects. + # This is done only at the user level. + 'groups': [], + 'people': user_cc_list + }, + 'owner': pagure_project['access_users']['owner'][0], + # No package has this set in PkgDB's API, so it can be safely turned + # off and set to the defaults later on in the code + 'qacontact': None, + 'summary': summary + } + + if __name__ == '__main__': sys.stdout = codecs.getwriter('utf-8')(sys.stdout) - parser = argparse.ArgumentParser( - description='Script syncing information between pkgdb and bugzilla' + description='Script syncing information between Pagure and bugzilla' ) parser.add_argument( '--debug', dest='debug', action='store_true', default=False, @@ -414,41 +494,93 @@ if __name__ == '__main__': # Non-fatal errors to alert people about errors = [] - # Get bugzilla information from the package database - req = requests.get('%s/api/bugzilla/?format=json' % PKGDBSERVER) - acls = req.json()['bugzillaAcls'] + projects_dict = { + 'Fedora': {}, + 'Fedora Container': {}, + 'Fedora EPEL': {}, + } + pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&' + 'per_page=100'.format(PAGUREURL.rstrip('/'))) + while True: + if DRY_RUN: + print('Querying {0}'.format(pagure_rpms_api_url)) + rv_json = requests.get(pagure_rpms_api_url, timeout=120).json() + for project in rv_json['projects']: + pagure_project_branches_api_url = ( + '{0}/api/0/rpms/{1}/git/branches' + .format(PAGUREURL.rstrip('/'), project['name'])) + branch_rv_json = requests.get( + pagure_project_branches_api_url, timeout=60).json() + project_pkgdb_schema = pagure_project_to_acl_schema(project) + epel = False + fedora = False + for branch in branch_rv_json['branches']: + if re.match(r'epel\d+', branch): + epel = True + projects_dict['Fedora EPEL'][project['name']] = \ + project_pkgdb_schema + else: + fedora = True + projects_dict['Fedora'][project['name']] = \ + project_pkgdb_schema + + if fedora and epel: + break + + if rv_json['pagination']['next']: + pagure_rpms_api_url = rv_json['pagination']['next'] + else: + break + + pagure_container_api_url = ( + '{0}/api/0/projects?&namespace=container&page=1&per_page=100' + .format(PAGUREURL)) + while True: + if DRY_RUN: + print('Querying {0}'.format(pagure_container_api_url)) + rv_json = requests.get(pagure_container_api_url, timeout=120).json() + for project in rv_json['projects']: + project_pkgdb_schema = pagure_project_to_acl_schema(project) + projects_dict['Fedora Container'][project['name']] = \ + project_pkgdb_schema + + if rv_json['pagination']['next']: + pagure_container_api_url = rv_json['pagination']['next'] + else: + break # Initialize the connection to bugzilla - bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, acls) + bugzilla = Bugzilla(BZSERVER, BZUSER, BZPASS, projects_dict) - for product in acls.keys(): + for product in projects_dict.keys(): if product not in PRODUCTS: continue - for pkg in sorted(acls[product]): + for pkg in sorted(projects_dict[product]): if DRY_RUN: - print pkg - pkgInfo = acls[product][pkg] + print(pkg) + pkgInfo = projects_dict[product][pkg] try: bugzilla.add_edit_component( - pkg, - product, - pkgInfo['owner'], - pkgInfo['summary'], - pkgInfo['qacontact'], - pkgInfo['cclist']) - except ValueError, e: + pkg, + product, + pkgInfo['owner'], + pkgInfo['summary'], + pkgInfo['qacontact'], + pkgInfo['cclist'] + ) + except ValueError as e: # A username didn't have a bugzilla address errors.append(str(e.args)) - except DataChangedError, e: + except DataChangedError as e: # A Package or Collection was returned via xmlrpc but wasn't # present when we tried to change it errors.append(str(e.args)) - except xmlrpclib.ProtocolError, e: + except xmlrpclib.ProtocolError as e: # Unrecoverable and likely means that nothing is going to # succeed. errors.append(str(e.args)) break - except xmlrpclib.Error, e: + except xmlrpclib.Error as e: # An error occurred in the xmlrpc call. Shouldn't happen but # we better see what it is errors.append('%s -- %s' % (pkg, e.args[-1])) @@ -456,7 +588,7 @@ if __name__ == '__main__': # Send notification of errors if errors: if DRY_RUN: - print '[DEBUG]', '\n'.join(errors) + print('[DEBUG]', '\n'.join(errors)) else: notify_users(errors) send_email( From 4783a88be8a40fd557271ffbf368cc40f540fd97 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Thu, 20 Jul 2017 20:25:53 +0000 Subject: [PATCH 07/17] Check for default assignee overrides --- roles/distgit/pagure/tasks/main.yml | 2 ++ .../templates/pagure-sync-bugzilla.py.j2 | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/roles/distgit/pagure/tasks/main.yml b/roles/distgit/pagure/tasks/main.yml index 3df3ba5d08..0cc8dd7c60 100644 --- a/roles/distgit/pagure/tasks/main.yml +++ b/roles/distgit/pagure/tasks/main.yml @@ -17,6 +17,8 @@ # For the pagure-sync-bugzilla.py script - python-bugzilla - python2-requests + - PyYAML + - python-six # - mod_ssl # - stunnel tags: diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index ceb6b86314..07d06af755 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -38,12 +38,15 @@ import json import xmlrpclib import codecs import smtplib -import bugzilla -import requests try: from email.Message import Message except ImportError: from email.message import EmailMessage as Message + +import bugzilla +import requests +import yaml +from six import string_types from fedora.client.fas2 import AccountSystem BZSERVER = 'https://bugzilla.redhat.com' @@ -419,7 +422,7 @@ def notify_users(errors): json.dump(new_data, stream) -def pagure_project_to_acl_schema(pagure_project): +def pagure_project_to_acl_schema(pagure_project, product): """ This function translates the JSON of a Pagure project to what PkgDB used to output in the Bugzilla API. @@ -461,6 +464,20 @@ def pagure_project_to_acl_schema(pagure_project): mdapi_url, mdapi_rv.status_code, mdapi_rv.text) raise RuntimeError(error_msg) + # Check if the Bugzilla ticket assignee has been overridden + owner = pagure_project['access_users']['owner'][0] + pagure_override_url = ( + 'https://pagure.io/bugzilla-assignee-override/raw/master/f/{0}/{1}' + .format(project['namespace'], project['name'])) + + override_rv = requests.get(pagure_override_url, timeout=30) + if override_rv.status_code == 200: + override_yaml = yaml.load(override_rv.text) + if override_yaml.get(product) \ + and isinstance(override_yaml[product], string_types) \ + and '@' in override_yaml[product]: + owner = override_yaml[product] + return { 'cclist': { # Groups is empty because you can't have groups watch projects. @@ -468,7 +485,7 @@ def pagure_project_to_acl_schema(pagure_project): 'groups': [], 'people': user_cc_list }, - 'owner': pagure_project['access_users']['owner'][0], + 'owner': owner, # No package has this set in PkgDB's API, so it can be safely turned # off and set to the defaults later on in the code 'qacontact': None, @@ -511,18 +528,17 @@ if __name__ == '__main__': .format(PAGUREURL.rstrip('/'), project['name'])) branch_rv_json = requests.get( pagure_project_branches_api_url, timeout=60).json() - project_pkgdb_schema = pagure_project_to_acl_schema(project) epel = False fedora = False for branch in branch_rv_json['branches']: if re.match(r'epel\d+', branch): epel = True projects_dict['Fedora EPEL'][project['name']] = \ - project_pkgdb_schema + pagure_project_to_acl_schema(project, 'epel') else: fedora = True projects_dict['Fedora'][project['name']] = \ - project_pkgdb_schema + pagure_project_to_acl_schema(project, 'fedora') if fedora and epel: break From 150dc4c5c5dc2b58940446f3cc99df8bc90b1582 Mon Sep 17 00:00:00 2001 From: Matt Prahl Date: Sun, 30 Jul 2017 01:31:19 +0000 Subject: [PATCH 08/17] Address feedback --- .../templates/pagure-sync-bugzilla.py.j2 | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 07d06af755..712ac77206 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -1,7 +1,7 @@ #!/usr/bin/python -tt # -*- coding: utf-8 -*- # -# Copyright © 2013-2016 Red Hat, Inc. +# Copyright © 2013-2017 Red Hat, Inc. # # This copyrighted material is made available to anyone wishing to use, modify, # copy, or redistribute it subject to the terms and conditions of the GNU @@ -19,6 +19,7 @@ # Red Hat Author(s): Toshio Kuratomi # Author(s): Mike Watters # Author(s): Pierre-Yves Chibon +# Author(s): Matt Prahl # ''' sync information from the Pagure into bugzilla @@ -55,6 +56,7 @@ BZPASS = '{{ bugzilla_password }}' BZCOMPAPI = 'component.get' FASUSER = '{{ fedorathirdpartyUser }}' FASPASS = '{{ fedorathirdpartyPassword }}' +BUGZILLA_OVERRIDE_REPO = 'dist-git-requests' NOTIFYEMAIL = [ 'kevin@fedoraproject.org', 'pingou@fedoraproject.org'] @@ -63,12 +65,14 @@ DRY_RUN = False {% if env == 'staging' %} FASURL = 'https://admin.stg.fedoraproject.org/accounts' FASINSECURE = True -PAGUREURL = 'https://src.stg.fedoraproject.org/pagure/' +PAGUREURL = 'https://stg.pagure.io' +PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org/pagure/' MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/' {% else %} FASURL = 'https://admin.fedoraproject.org/accounts' FASINSECURE = False -PAGUREURL = 'https://src.fedoraproject.org/pagure/' +PAGUREURL = 'https://pagure.io' +PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org/pagure/' MDAPIURL = 'https://apps.fedoraproject.org/mdapi/' {% endif %} @@ -158,7 +162,7 @@ class ProductCache(dict): if BZCOMPAPI == 'getcomponentsdetails': # Old API -- in python-bugzilla. But with current server, this # gives ProxyError - products = self.server.getcomponentsdetails(key) + products = self.bz.getcomponentsdetails(key) elif BZCOMPAPI == 'component.get': # Way that's undocumented in the partner-bugzilla api but works # currently @@ -432,7 +436,7 @@ def pagure_project_to_acl_schema(pagure_project, product): base_error_msg = ('The connection to "{0}" failed with the status code ' '{1} and output "{2}"') watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format( - PAGUREURL.rstrip('/'), pagure_project['namespace'], + PAGURE_DIST_GIT_URL.rstrip('/'), pagure_project['namespace'], pagure_project['name']) if DRY_RUN: print('Querying {0}'.format(watchers_api_url)) @@ -466,16 +470,15 @@ def pagure_project_to_acl_schema(pagure_project, product): # Check if the Bugzilla ticket assignee has been overridden owner = pagure_project['access_users']['owner'][0] - pagure_override_url = ( - 'https://pagure.io/bugzilla-assignee-override/raw/master/f/{0}/{1}' - .format(project['namespace'], project['name'])) + pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format( + PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'], + project['name']) override_rv = requests.get(pagure_override_url, timeout=30) if override_rv.status_code == 200: override_yaml = yaml.load(override_rv.text) if override_yaml.get(product) \ - and isinstance(override_yaml[product], string_types) \ - and '@' in override_yaml[product]: + and isinstance(override_yaml[product], string_types): owner = override_yaml[product] return { @@ -517,7 +520,8 @@ if __name__ == '__main__': 'Fedora EPEL': {}, } pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&' - 'per_page=100'.format(PAGUREURL.rstrip('/'))) + 'per_page=100'.format( + PAGURE_DIST_GIT_URL.rstrip('/'))) while True: if DRY_RUN: print('Querying {0}'.format(pagure_rpms_api_url)) @@ -525,7 +529,7 @@ if __name__ == '__main__': for project in rv_json['projects']: pagure_project_branches_api_url = ( '{0}/api/0/rpms/{1}/git/branches' - .format(PAGUREURL.rstrip('/'), project['name'])) + .format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name'])) branch_rv_json = requests.get( pagure_project_branches_api_url, timeout=60).json() epel = False @@ -534,11 +538,11 @@ if __name__ == '__main__': if re.match(r'epel\d+', branch): epel = True projects_dict['Fedora EPEL'][project['name']] = \ - pagure_project_to_acl_schema(project, 'epel') + pagure_project_to_acl_schema(project, 'Fedora EPEL') else: fedora = True projects_dict['Fedora'][project['name']] = \ - pagure_project_to_acl_schema(project, 'fedora') + pagure_project_to_acl_schema(project, 'Fedora') if fedora and epel: break @@ -550,7 +554,7 @@ if __name__ == '__main__': pagure_container_api_url = ( '{0}/api/0/projects?&namespace=container&page=1&per_page=100' - .format(PAGUREURL)) + .format(PAGURE_DIST_GIT_URL)) while True: if DRY_RUN: print('Querying {0}'.format(pagure_container_api_url)) From fd4a522b53fe85769c80864b84c3688153ebc410 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 3 Aug 2017 14:41:12 +0000 Subject: [PATCH 09/17] Adjust override yaml structure. --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 712ac77206..47c8948ee5 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -477,6 +477,7 @@ def pagure_project_to_acl_schema(pagure_project, product): override_rv = requests.get(pagure_override_url, timeout=30) if override_rv.status_code == 200: override_yaml = yaml.load(override_rv.text) + override_yaml = override_yaml.get('bugzilla_contact', {}) if override_yaml.get(product) \ and isinstance(override_yaml[product], string_types): owner = override_yaml[product] From 09af2c0457780fbc02e7e240fed1626d4589ec55 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 3 Aug 2017 16:05:39 +0000 Subject: [PATCH 10/17] Only people watching issues should be CC'd --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 47c8948ee5..7c0de1a535 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -449,8 +449,8 @@ def pagure_project_to_acl_schema(pagure_project, product): user_cc_list = [] for user, watch_levels in watchers_rv_json['watchers'].items(): - # Only people watching commits should be CC'd - if 'commit' in watch_levels: + # Only people watching issues should be CC'd + if 'issues' in watch_levels: user_cc_list.append(user) summary = None From 78f8cbf7996dd3a160bd0fa217db0df12f43df3f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 9 Aug 2017 14:00:40 +0000 Subject: [PATCH 11/17] Fix src.fp.o URLs. --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 7c0de1a535..11ad3be74d 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -66,13 +66,13 @@ DRY_RUN = False FASURL = 'https://admin.stg.fedoraproject.org/accounts' FASINSECURE = True PAGUREURL = 'https://stg.pagure.io' -PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org/pagure/' +PAGURE_DIST_GIT_URL = 'https://src.stg.fedoraproject.org' MDAPIURL = 'https://apps.stg.fedoraproject.org/mdapi/' {% else %} FASURL = 'https://admin.fedoraproject.org/accounts' FASINSECURE = False PAGUREURL = 'https://pagure.io' -PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org/pagure/' +PAGURE_DIST_GIT_URL = 'https://src.fedoraproject.org' MDAPIURL = 'https://apps.fedoraproject.org/mdapi/' {% endif %} From ff3b5478f5e9b16b4d9fec80ff31c80e45a899f4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 9 Aug 2017 14:01:00 +0000 Subject: [PATCH 12/17] Use the official override repo. --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 11ad3be74d..4df88c7dde 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -56,7 +56,7 @@ BZPASS = '{{ bugzilla_password }}' BZCOMPAPI = 'component.get' FASUSER = '{{ fedorathirdpartyUser }}' FASPASS = '{{ fedorathirdpartyPassword }}' -BUGZILLA_OVERRIDE_REPO = 'dist-git-requests' +BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests' NOTIFYEMAIL = [ 'kevin@fedoraproject.org', 'pingou@fedoraproject.org'] From 27c8f24930019d6ad1be5b83e455dbefe30c7385 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 9 Aug 2017 14:03:38 +0000 Subject: [PATCH 13/17] Include more devs in error emails. --- roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 4df88c7dde..630455e91c 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -59,7 +59,10 @@ FASPASS = '{{ fedorathirdpartyPassword }}' BUGZILLA_OVERRIDE_REPO = 'releng/fedora-scm-requests' NOTIFYEMAIL = [ 'kevin@fedoraproject.org', - 'pingou@fedoraproject.org'] + 'pingou@fedoraproject.org', + 'ralph@fedoraproject.org', + 'mprahl@fedoraproject.org', +] DRY_RUN = False {% if env == 'staging' %} From 6b67be1f420466b89933fdff8c4b3b07f901ac1d Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 9 Aug 2017 14:09:56 +0000 Subject: [PATCH 14/17] Use requests retry mechanism in the bugzilla sync script. --- .../templates/pagure-sync-bugzilla.py.j2 | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 index 630455e91c..863bcd6542 100644 --- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 +++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 @@ -50,6 +50,24 @@ import yaml from six import string_types from fedora.client.fas2 import AccountSystem +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + + +def retry_session(): + session = requests.Session() + retry = Retry( + total=5, + read=5, + connect=5, + backoff_factor=0.3, + status_forcelist=(500, 502, 504), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + return session + BZSERVER = 'https://bugzilla.redhat.com' BZUSER = '{{ bugzilla_user }}' BZPASS = '{{ bugzilla_password }}' @@ -436,6 +454,7 @@ def pagure_project_to_acl_schema(pagure_project, product): :param pagure_project: a dictionary of the JSON of a Pagure project :return: a dictionary of the content that the Bugzilla API would output """ + session = retry_session() base_error_msg = ('The connection to "{0}" failed with the status code ' '{1} and output "{2}"') watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format( @@ -443,7 +462,7 @@ def pagure_project_to_acl_schema(pagure_project, product): pagure_project['name']) if DRY_RUN: print('Querying {0}'.format(watchers_api_url)) - watchers_rv = requests.get(watchers_api_url, timeout=60) + watchers_rv = session.get(watchers_api_url, timeout=60) if not watchers_rv.ok: error_msg = base_error_msg.format( watchers_api_url, watchers_rv.status_code, watchers_rv.text) @@ -462,7 +481,7 @@ def pagure_project_to_acl_schema(pagure_project, product): MDAPIURL.rstrip('/'), pagure_project['name']) if DRY_RUN: print('Querying {0}'.format(mdapi_url)) - mdapi_rv = requests.get(mdapi_url, timeout=60) + mdapi_rv = session.get(mdapi_url, timeout=60) if mdapi_rv.ok: mdapi_rv_json = mdapi_rv.json() summary = mdapi_rv_json['summary'] @@ -477,7 +496,7 @@ def pagure_project_to_acl_schema(pagure_project, product): PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'], project['name']) - override_rv = requests.get(pagure_override_url, timeout=30) + override_rv = session.get(pagure_override_url, timeout=30) if override_rv.status_code == 200: override_yaml = yaml.load(override_rv.text) override_yaml = override_yaml.get('bugzilla_contact', {}) @@ -526,15 +545,17 @@ if __name__ == '__main__': pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&' 'per_page=100'.format( PAGURE_DIST_GIT_URL.rstrip('/'))) + session = retry_session() + while True: if DRY_RUN: print('Querying {0}'.format(pagure_rpms_api_url)) - rv_json = requests.get(pagure_rpms_api_url, timeout=120).json() + rv_json = session.get(pagure_rpms_api_url, timeout=120).json() for project in rv_json['projects']: pagure_project_branches_api_url = ( '{0}/api/0/rpms/{1}/git/branches' .format(PAGURE_DIST_GIT_URL.rstrip('/'), project['name'])) - branch_rv_json = requests.get( + branch_rv_json = session.get( pagure_project_branches_api_url, timeout=60).json() epel = False fedora = False @@ -562,7 +583,7 @@ if __name__ == '__main__': while True: if DRY_RUN: print('Querying {0}'.format(pagure_container_api_url)) - rv_json = requests.get(pagure_container_api_url, timeout=120).json() + rv_json = session.get(pagure_container_api_url, timeout=120).json() for project in rv_json['projects']: project_pkgdb_schema = pagure_project_to_acl_schema(project) projects_dict['Fedora Container'][project['name']] = \ From 6aa24d1036c2417db5590d8c79602a018827f646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20P=C3=A1ral?= Date: Wed, 9 Aug 2017 16:51:13 +0200 Subject: [PATCH 15/17] taskotron-stg: try to fix critpath url --- inventory/group_vars/taskotron-stg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/taskotron-stg b/inventory/group_vars/taskotron-stg index 92cb34710f..287c69bd85 100644 --- a/inventory/group_vars/taskotron-stg +++ b/inventory/group_vars/taskotron-stg @@ -53,7 +53,7 @@ resultsdb_url: http://resultsdb-stg01.qa.fedoraproject.org/resultsdb_api/api/v2. taskotron_docs_url: https://qa.fedoraproject.org/docs/libtaskotron/latest/ # make sure this doesn't have a trailing slash trigger_distgit_repo_url: git://pkgs02.phx2.fedoraproject.org -trigger_critpath_url: https://admin.fedoraproject.org/pkgdb/api/critpath?format=json +trigger_critpath_url: https://admin.stg.fedoraproject.org/pkgdb/api/critpath?format=json instances: - { name: "Production", url: "https://taskotron.fedoraproject.org"} From 5b433d2a201ba472140d8ed0a07b9fd38bf5223c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 9 Aug 2017 18:55:04 +0200 Subject: [PATCH 16/17] Tweak pagure config on dist-git --- roles/distgit/pagure/templates/pagure.cfg | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/distgit/pagure/templates/pagure.cfg b/roles/distgit/pagure/templates/pagure.cfg index ab5ca71493..1c574a411e 100644 --- a/roles/distgit/pagure/templates/pagure.cfg +++ b/roles/distgit/pagure/templates/pagure.cfg @@ -182,7 +182,7 @@ SSH_KEYS = { # Configuration item that are specific for this odd pagure instance PROJECT_TICKETS = False -ENABLE_NEW_PROJECTS = False +ENABLE_NEW_PROJECTS = True ENABLE_DEL_PROJECTS = False ENABLE_TICKETS = False ENABLE_GROUP_MNGT = False @@ -222,6 +222,7 @@ PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1/' {% endif %} GITOLITE_BACKEND = 'distgit' +GITOLITE_CELERY_QUEUE = 'gitolite_queue' THEME_TEMPLATE_FOLDER = '/usr/share/pagure_dist_git/template/' @@ -244,6 +245,9 @@ ADMIN_API_ACLS = [ 'pull_request_comment', 'pull_request_merge', 'create_project', + 'modify_project', ] BLACKLISTED_GROUPS = ['forks', 'group'] + + From e7b1d708fbd031c496b67d770cab8ed78023d658 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 9 Aug 2017 18:57:43 +0200 Subject: [PATCH 17/17] python2-requests doesn't exist in epel7 --- roles/distgit/pagure/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/distgit/pagure/tasks/main.yml b/roles/distgit/pagure/tasks/main.yml index 0cc8dd7c60..42c7087677 100644 --- a/roles/distgit/pagure/tasks/main.yml +++ b/roles/distgit/pagure/tasks/main.yml @@ -16,7 +16,7 @@ - python2-pagure-dist-git # For the pagure-sync-bugzilla.py script - python-bugzilla - - python2-requests + - python-requests - PyYAML - python-six # - mod_ssl