From 51842f16481918447f593b1df4607f4cc7b82400 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 13 Jul 2017 17:52:32 +0200 Subject: [PATCH] Drop genacls.pkgdb in stg --- roles/distgit/tasks/main.yml | 10 - roles/distgit/templates/genacls.pkgdb.stg | 465 ---------------------- 2 files changed, 475 deletions(-) delete mode 100644 roles/distgit/templates/genacls.pkgdb.stg diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 0cb1c93b38..752844a94c 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -259,16 +259,6 @@ - config - distgit -- name: install the genacls.pkgdb scripts - template: src={{item}} dest=/usr/local/bin/genacls.pkgdb - owner=root group=root mode=0755 - with_items: - - genacls.pkgdb.stg - when: env == "staging" - tags: - - config - - distgit - - name: Add the genacl daily cron job copy: src=genacls.cron dest=/etc/cron.d/genacls.cron owner=root mode=644 diff --git a/roles/distgit/templates/genacls.pkgdb.stg b/roles/distgit/templates/genacls.pkgdb.stg deleted file mode 100644 index 5673aca445..0000000000 --- a/roles/distgit/templates/genacls.pkgdb.stg +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/python -t -# -# Create an /etc/gitolite/conf/gitolite.conf file with acls for dist-git -# -# Takes no arguments! -# - -import copy -import grp -import itertools -import os -import sys -import json - -from multiprocessing import Pool, Manager - -import requests -from sqlalchemy.exc import SQLAlchemyError - - -TESTING = False - - -if 'PAGURE_CONFIG' not in os.environ \ - and os.path.exists('/etc/pagure/pagure.cfg'): - if TESTING: - print 'Using configuration file `/etc/pagure/pagure.cfg`' - os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg' - -import pagure -from pagure import SESSION -from pagure.exceptions import PagureException - -{% if env == 'staging' %} -VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json' -GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json' -{% else %} -VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json' -GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json' -{% endif %} - - -RELENG_USER = { - 'username': 'releng', - 'fullname': 'Fedora Release Engineering', - 'default_email': 'releng-team@fedoraproject.org', -} - - -def get_user_info(username): - ''' Uses python-fedora to get information about a FAS user ''' - - user = { - 'username': username, - 'fullname': username, - 'default_email': '%s@fedoraproject.org' % username - } - return user - - -def create_user_obj(session, username, userinfo=None): - ''' Creates a sqlalchemy user object for pagure db ''' - user = None - try: - if not userinfo: - userinfo = get_user_info(username) - user = pagure.lib.set_up_user( - session=session, - username=username, - fullname=userinfo['fullname'], - default_email=userinfo['default_email'] - ) - session.commit() - except SQLAlchemyError: - session.rollback() - if TESTING: - print 'Creating user failed' - - return user - - -def create_groups_in_db(groups): - ''' Creates groups in pagure db ''' - - group_keys = groups.keys() - for groupname in group_keys: - - # we don't need to do anything with empty groups, do we? - if len(groups[groupname]) == 0: - continue - - # first make sure that the users in the groups are present in the db - group_users = groups[groupname] - for guser in group_users: - user_obj = pagure.lib.search_user(SESSION, username=guser) - if not user_obj: - user_obj = create_user_obj(SESSION, guser) - - # check if the groups are present in the db - group_obj = pagure.lib.search_groups(SESSION, group_name=groupname) - if not group_obj: - # add the group to the db using the first user in the group - try: - pagure.lib.add_group( - session=SESSION, - group_name=groupname, - display_name=groupname, - description=None, - group_type='user', - user='releng', - is_admin=False, - blacklist=pagure.APP.config['BLACKLISTED_GROUPS'] - ) - SESSION.commit() - except SQLAlchemyError: - SESSION.rollback() - if TESTING: - print 'Adding a user to group failed' - - # now that all groups are present in the db - # make sure that all the members are there in the group in the db - for guser in group_users: - if not pagure.lib.is_group_member(SESSION, guser, groupname): - group_obj = pagure.lib.search_groups( - session=SESSION, - group_name=groupname - ) - try: - pagure.lib.add_user_to_group( - session=SESSION, - username=guser, - group=group_obj, - user='releng', - is_admin=False - ) - SESSION.commit() - except SQLAlchemyError: - SESSION.rollback() - if TESTING: - print 'Adding a user to group failed' - - -def update_owners_to_db(session, namespace, pkg, owners): - ''' Adds owners to pagure db ''' - pkg_obj = pagure.lib.get_project( - session, name=pkg, namespace=namespace) - - for owner in owners: - # check if the owners are present in the db - # if not create them - owner_obj = pagure.lib.search_user(session, username=owner) - if not owner_obj: - owner_obj = create_user_obj(session, owner) - - - # this flag is for avoiding unnecessary db queries - created = False - if not pkg_obj: - try: - pagure.lib.new_project( - session=session, - user='releng', - namespace=namespace, - name=pkg, - blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'], - allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'], - gitfolder=pagure.APP.config['GIT_FOLDER'], - docfolder=pagure.APP.config['DOCS_FOLDER'], - ticketfolder=pagure.APP.config['TICKETS_FOLDER'], - requestfolder=pagure.APP.config['REQUESTS_FOLDER'], - ignore_existing_repo=True, - ) - session.commit() - created = True - except SQLAlchemyError as err: - session.rollback() - if TESTING: - print "Couldn't create project - %s" % pkg - print "ERROR: %s" % err - except PagureException as err: - if TESTING: - print "Couldn't create project - %s" % pkg - print "ERROR: %s" % err - - # so now the pkg surely exists, make the owner, - # the owner of the repo if he is not - if created: - pkg_obj = pagure.lib.get_project( - session=session, - name=pkg, - namespace=namespace - ) - - #if owner_obj not in pkg_obj.admins and owner_obj is not pkg_obj.user: - if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user: - try: - pagure.lib.add_user_to_project( - session=session, - project=pkg_obj, - new_user=owner_obj.user, - user='releng', - access='commit', - ) - session.commit() - except SQLAlchemyError as err: - SESSION.rollback() - if TESTING: - print "Couldn't add user to project" - print "ERROR: %s" % err - - -def update_groups_to_db(session, namespace, pkg, pkg_groups): - ''' Adds groups to projects in pagure db ''' - - pkg_obj = pagure.lib.get_project( - session, name=pkg, namespace=namespace) - - for group in pkg_groups: - # we have already created all the groups - group_obj = pagure.lib.search_groups(session, group_name=group) - - # in case when there are only groups with commit access and no - # people the flag is for cutting out db queries later - created = False - if not pkg_obj: - try: - pagure.lib.new_project( - session=session, - user='releng', - namespace=namespace, - name=pkg, - blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'], - allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'], - gitfolder=pagure.APP.config['GIT_FOLDER'], - docfolder=pagure.APP.config['DOCS_FOLDER'], - ticketfolder=pagure.APP.config['TICKETS_FOLDER'], - requestfolder=pagure.APP.config['REQUESTS_FOLDER'], - ignore_existing_repo=True, - ) - session.commit() - created = True - except SQLAlchemyError as err: - session.rollback() - if TESTING: - print "Couldn't create project" - print "ERROR: %s" % err - except PagureException as err: - if TESTING: - print "Couldn't create project - %s" % pkg - print "ERROR: %s" % err - - # for the case when the new project was just created - # by the above call - if created: - pkg_obj = pagure.lib.get_project( - session, name=pkg, namespace=namespace) - - # if the group was initially empty, it was not - # created in the db - if not group_obj: - continue - - # check if the group is added to project - # if not, add them - if group_obj not in pkg_obj.groups: - try: - pagure.lib.add_group_to_project( - session=session, - project=pkg_obj, - new_group=group, - user='releng', - access='commit', - ) - session.commit() - except SQLAlchemyError as err: - session.rollback() - if TESTING: - print "Adding a group to a project failed" - print "ERROR: %s" % err - - -def add_fork_to_gitolite(): - ''' Creates a sqlalchemy user object for pagure db ''' - for fork in pagure.lib.search_projects(session=SESSION, fork=True): - print '' - print 'repo %s' % (fork.fullname) - print ' RWC = %s' % fork.user.username - - -def process_pkg(arg): - """ Process the given package, adjust pagure for it and queue all the - ACLs so we can send them to gitolite - """ - pkg, acls, myq = arg - session = pagure.lib.create_session(pagure.APP.config['DB_URL']) - - branchAcls = {} # Check whether we need to set separate per branch acls - buffer = [] # Buffer the output per package - masters = [] # Folks that have commit to master - writers = [] # Anybody that has write access - - # Examine each branch in the package - branches = acls[pkg].keys() - branches.sort() - for branch in branches: - if branch not in ACTIVE.keys(): - continue - if 'packager' in acls[pkg][branch]['commit']['groups']: - # If the packager group is defined, everyone has access - buffer.append(' RWC %s = @all' % (ACTIVE[branch])) - branchAcls.setdefault('@all', []).append( - (pkg, ACTIVE[branch]) - ) - if branch == 'master': - masters.append('@all') - if '@all' not in writers: - writers.append('@all') - else: - # Extract the owners - committers = [] - owners = acls[pkg][branch]['commit']['people'] - owners.sort() - for owner in owners: - committers.append(owner) - for group in acls[pkg][branch]['commit']['groups']: - committers.append('@%s' % group) - if branch == 'master': - masters.extend(committers) - - pkg_groups = acls[pkg][branch]['commit']['groups'] - update_owners_to_db(session, namespace, pkg, owners) - update_groups_to_db(session, namespace, pkg, pkg_groups) - - # add all the committers to the top writers list - for committer in committers: - if committer not in writers: - writers.append(committer) - - # Print the committers to the acl for this package-branch - committers = ' '.join(committers) - buffer.append( - ' RWC %s = %s' % (ACTIVE[branch], committers)) - branchAcls.setdefault(committers, []).append( - (pkg, ACTIVE[branch]) - ) - - session.close() - data = [pkg, buffer, writers, masters] - myq.put(data) - myq.task_done() - - -if __name__ == '__main__': - - # Create the rel-eng user if it needs to - if not pagure.lib.search_user(SESSION, username='releng'): - create_user_obj(SESSION, 'releng', RELENG_USER) - - TRUSTED = grp.getgrnam('cvsadmin')[3] - ARM = grp.getgrnam('fedora-arm')[3] - SPARC = grp.getgrnam('fedora-sparc')[3] - IA64 = grp.getgrnam('fedora-ia64')[3] - S390 = grp.getgrnam('fedora-s390')[3] - PPC = grp.getgrnam('fedora-ppc')[3] - PROVEN = grp.getgrnam('provenpackager')[3] - - # Set the active branches to create ACLs for - # Give them the git branch eqiv until pkgdb follows suite - ACTIVE = { - 'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4', - 'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6', - 'epel7': 'epel7', - 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': - 'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19', - 'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24', - 'f25': 'f25', - 'devel': 'master', 'master': 'master'} - - # Create a "regex"ish list 0f the reserved branches - RESERVED = [ - 'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]', - 'olpc[0-9]'] - - # print out our user groups - print '@admins = %s' % ' '.join(TRUSTED) - print '@provenpackager = %s' % ' '.join(PROVEN) - print '@fedora-arm = %s' % ' '.join(ARM) - print '@fedora-s390 = %s' % ' '.join(S390) - print '@fedora-ppc = %s' % ' '.join(PPC) - - groups = { - 'admins': TRUSTED, - 'fedora-arm': ARM, - 'SPARC': SPARC, - 'IA65': IA64, - 'fedora-s390': S390, - 'fedora-ppc': PPC, - 'provenpackager': PROVEN - } - - # Get a list of all the groups - groups_ = requests.get(GRP_URL).json() - for group in groups_['groups']: - print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3])) - gmems = grp.getgrnam(group)[3] - if group not in groups.keys(): - groups[group] = gmems - elif groups[group] != gmems: - groups[group] = gmems - - create_groups_in_db(groups) - - # Check the blacklist and if the name clashes - # append '-1' after them - tmp workaround - #blacklist = pagure.APP.config.get('BLACKLISTED_PROJECTS') - #pkgs_list = data['rpms'].keys() - #for i in pkgs_list: - #if i in blacklist: - #data['rpms'][i + '-1'] = data['rpms'].pop(i) - - data = requests.get(VCS_URL).json() - - # Give a little space before moving onto the permissions - print '' - # print our default permissions - print 'repo @all' - print ' - VREF/update-block-push-origin = @all' - print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc' - print ' R = @all' - #print ' RW private- = @all' - # dont' enable the above until we prevent building for real from private- - - # Get a list of all the packages - for namespace in data: - if namespace == 'title': - continue - - acls = data[namespace] - pkglist = sorted(data[namespace].keys()) - - m = Manager() - q = m.Queue() - p = Pool(5) - p.map(process_pkg, itertools.product(pkglist, [acls], [q])) - p.close() - p.join() - - #for pkg in pkglist: - #process_pkg([pkg, acls, q]) - while q.qsize(): - pkg, buffer, writers, masters = q.get() - print '' - print 'repo %s/%s' % (namespace, pkg) - print '\n'.join(buffer) - for reserved in RESERVED: - print ' - %s = @all' % reserved - print ' RWC refs/tags/ = %s' % ' '.join(writers) - if masters: - print ' RWC = %s' % ' '.join(masters) - - q.join() - - add_fork_to_gitolite() - - sys.exit(0)