From de67bcbea22bb4539e32d195a10448948bc6d765 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Sat, 30 Jul 2016 10:42:50 +0200 Subject: [PATCH] Install a different genacls.pkgdb in stg vs prod now that stg has pagure --- roles/distgit/tasks/main.yml | 11 + roles/distgit/templates/genacls.pkgdb.stg | 366 ++++++++++++++++++++++ 2 files changed, 377 insertions(+) create mode 100644 roles/distgit/templates/genacls.pkgdb.stg diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 132cf55d3d..499def547c 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -186,6 +186,17 @@ owner=root group=root mode=0755 with_items: - genacls.pkgdb + when: env != "staging" + tags: + - config + - distgit + +- name: install the genacls.pkgdb scripts + template: src={{item}} dest=/usr/local/bin/genacls.pkgdb + owner=root group=root mode=0755 + with_items: + - genacls.pkgdb.stg + when: env == "staging" tags: - config - distgit diff --git a/roles/distgit/templates/genacls.pkgdb.stg b/roles/distgit/templates/genacls.pkgdb.stg new file mode 100644 index 0000000000..44018798c4 --- /dev/null +++ b/roles/distgit/templates/genacls.pkgdb.stg @@ -0,0 +1,366 @@ +#!/usr/bin/python -t +# +# Create an /etc/gitolite/conf/gitolite.conf file with acls for dist-git +# +# Takes no arguments! +# + +import copy +import grp +import sys +import json + +import requests +from sqlalchemy.exc import SQLAlchemyError + +import pagure +from pagure import SESSION + +TESTING = False + +{% if env == 'staging' %} +VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json' +GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json' +{% else %} +VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json' +GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json' +{% endif %} + + +def create_user_obj(username): + ''' Creates a sqlalchemy user object for pagure db ''' + try: + user = pagure.lib.set_up_user( + SESSION, + username, + username, + '%s@fedorahosted.org' % username + ) + SESSION.commit() + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print 'Creating user failed' + + return user + + +def create_groups_in_db(groups): + ''' Creates groups in pagure db ''' + + group_keys = groups.keys() + for groupname in group_keys: + + # we don't need to do anything with empty groups, do we? + if len(groups[groupname]) == 0: + continue + + # first insure the users in the groups are present in the db + group_users = groups[groupname] + for guser in group_users: + user_obj = pagure.lib.search_user(SESSION, username=guser) + if not user_obj: + user_obj = create_user_obj(guser) + + # check if the groups are present in the db + group_obj = pagure.lib.search_groups(SESSION, group_name=groupname) + if not group_obj: + # add the group to the db using the first user in the group + try: + pagure.lib.add_group( + SESSION, groupname, 'user', + groups[groupname][0], False, + pagure.APP.config['BLACKLISTED_GROUPS']) + SESSION.commit() + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print 'Adding a user to group failed' + + # now that all groups are present in the db + # ensure all the members are there in the group in the db + for guser in group_users: + if not pagure.lib.is_group_member(SESSION, guser, groupname): + group_obj = pagure.lib.search_groups( + SESSION, group_name=groupname) + try: + msg = pagure.lib.add_user_to_group( + SESSION, + guser, + group_obj, + groups[groupname][0], + False + ) + SESSION.commit() + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print 'Adding a user to group failed' + + +def update_owners_to_db(pkg, owners): + ''' Adds owners to pagure db ''' + + for owner in owners: + # check if the owners are present in the db if not create them + owner_obj = pagure.lib.search_user(SESSION, username=owner) + if not owner_obj: + owner_obj = create_user_obj(owner) + + # check if the repo exists, if not create + pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) + + # this flag is for avoiding unnecessary db queries + flag = True + if not pkg_obj: + pkgname = pkg + try: + msg = pagure.lib.new_project( + SESSION, + owner, + 'rpms/%s' % pkgname, + pagure.APP.config['BLACKLISTED_PROJECTS'], + pagure.APP.config['ALLOWED_PREFIX'], + pagure.APP.config['GIT_FOLDER'], + pagure.APP.config['DOCS_FOLDER'], + pagure.APP.config['TICKETS_FOLDER'], + pagure.APP.config['REQUESTS_FOLDER'], + ) + SESSION.commit() + flag = False + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print "Couldn't create project" + + # so now the pkg surely exists, make the owner, the owner of the repo + # if (s)he is not + if not flag: + pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) + + if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user: + try: + msg = pagure.lib.add_user_to_project( + SESSION, + pkg_obj, + owner_obj.user, + pkg_obj.user.user, + ) + SESSION.commit() + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print "Couldn't add user to project" + + +def update_groups_to_db(pkg, pkg_groups): + ''' Adds groups to projects in pagure db ''' + + for group in pkg_groups: + # we have already created all the groups + group_obj = pagure.lib.search_groups(SESSION, group_name=group) + pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) + + # In case when there are only groups with commit access and no people + # The below flag is for cutting out db queries later + flag = True + if not pkg_obj: + pkgname = pkg + try: + msg = pagure.lib.new_project( + SESSION, + group_obj.creator.user, + 'rpms/%s' % pkgname, + pagure.APP.config['BLACKLISTED_PROJECTS'], + pagure.APP.config['ALLOWED_PREFIX'], + pagure.APP.config['GIT_FOLDER'], + pagure.APP.config['DOCS_FOLDER'], + pagure.APP.config['TICKETS_FOLDER'], + pagure.APP.config['REQUESTS_FOLDER'], + ) + SESSION.commit() + flag = False + except SQLAlchemyError: + SESSION.rollback() + if TESTING: + print "Couldn't create project" + + # for the case when the new project was just created by the above call + if not flag: + pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) + + # if the group was initially empty, it was not created in the db + if not group_obj: + continue + + # check if the group is added to project if not, add them + if group_obj not in pkg_obj.groups: + group_creator = group_obj.creator + try: + pagure.lib.add_group_to_project( + SESSION, + pkg_obj, + group, + pkg_obj.user.user, + ) + SESSION.commit() + except SQLAlchemyError as err: + SESSION.rollback() + if TESTING: + print "Adding a group to a project failed" + + +if __name__ == '__main__': + + TRUSTED = grp.getgrnam('cvsadmin')[3] + ARM = grp.getgrnam('fedora-arm')[3] + SPARC = grp.getgrnam('fedora-sparc')[3] + IA64 = grp.getgrnam('fedora-ia64')[3] + S390 = grp.getgrnam('fedora-s390')[3] + PPC = grp.getgrnam('fedora-ppc')[3] + PROVEN = grp.getgrnam('provenpackager')[3] + + # Set the active branches to create ACLs for + # Give them the git branch eqiv until pkgdb follows suite + ACTIVE = { + 'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4', + 'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6', + 'epel7': 'epel7', + 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': + 'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19', + 'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24', + 'f25': 'f25', + 'devel': 'master', 'master': 'master'} + + # Create a "regex"ish list 0f the reserved branches + RESERVED = [ + 'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]', + 'olpc[0-9]'] + + # print out our user groups + print '@admins = %s' % ' '.join(TRUSTED) + print '@provenpackager = %s' % ' '.join(PROVEN) + print '@fedora-arm = %s' % ' '.join(ARM) + print '@fedora-s390 = %s' % ' '.join(S390) + print '@fedora-ppc = %s' % ' '.join(PPC) + + groups = { + 'admins': TRUSTED, + 'fedora-arm': ARM, + 'SPARC': SPARC, + 'IA65': IA64, + 'fedora-s390': S390, + 'fedora-ppc': PPC, + 'provenpackager': PROVEN + } + + # Get a list of all the groups + groups_ = requests.get(GRP_URL).json() + for group in groups_['groups']: + print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3])) + gmems = grp.getgrnam(group)[3] + if group not in groups.keys(): + groups[group] = gmems + elif groups[group] != gmems: + groups[group] = gmems + + # + create_groups_in_db(groups) + # + + data = requests.get(VCS_URL).json() + + # Give a little space before moving onto the permissions + print '' + # print our default permissions + print 'repo @all' + print ' - VREF/update-block-push-origin = @all' + print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc' + print ' R = @all' + #print ' RW private- = @all' + # dont' enable the above until we prevent building for real from private- + + # XXX - Insert an artificial namespace into the set of namespaces returned + # by pkgdb. We want to create a mirror of rpms/PKG in rpms-checks/PKG + # This hack occurs in two places. Here, and in the branch-creation script. + # https://github.com/fedora-infra/pkgdb2/issues/329#issuecomment-207050233 + data['rpms-checks'] = copy.copy(data['rpms']) + + # Get a list of all the packages + for key in data: + if key == 'title': + continue + + acls = data[key] + pkglist = data[key].keys() + pkglist.sort() + + if key != 'packageAcls': + key = '%s/' % key + else: + key = '' + + for pkg in pkglist: + + branchAcls = {} # Check whether we need to set separate per branch acls + buffer = [] # Buffer the output per package + masters = [] # Folks that have commit to master + writers = [] # Anybody that has write access + + # Examine each branch in the package + branches = acls[pkg].keys() + branches.sort() + for branch in branches: + if branch not in ACTIVE.keys(): + continue + if 'packager' in acls[pkg][branch]['commit']['groups']: + # If the packager group is defined, everyone has access + buffer.append(' RWC %s = @all' % (ACTIVE[branch])) + branchAcls.setdefault('@all', []).append( + (pkg, ACTIVE[branch]) + ) + if branch == 'master': + masters.append('@all') + if '@all' not in writers: + writers.append('@all') + else: + # Extract the owners + committers = [] + owners = acls[pkg][branch]['commit']['people'] + owners.sort() + for owner in owners: + committers.append(owner) + for group in acls[pkg][branch]['commit']['groups']: + committers.append('@%s' % group) + if branch == 'master': + masters.extend(committers) + + # + pkg_groups = acls[pkg][branch]['commit']['groups'] + update_owners_to_db(pkg, owners) + update_groups_to_db(pkg, pkg_groups) + # + + # add all the committers to the top writers list + for committer in committers: + if committer not in writers: + writers.append(committer) + + # Print the committers to the acl for this package-branch + committers = ' '.join(committers) + buffer.append( + ' RWC %s = %s' % (ACTIVE[branch], committers)) + branchAcls.setdefault(committers, []).append( + (pkg, ACTIVE[branch]) + ) + + print '' + print 'repo %s%s' % (key, pkg) + print '\n'.join(buffer) + for reserved in RESERVED: + print ' - %s = @all' % reserved + print ' RWC refs/tags/ = %s' % ' '.join(writers) + if masters: + print ' RWC = %s' % ' '.join(masters) + + sys.exit(0)