Infrastructure/ansible
3
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Drop genacls.pkgdb in stg

Browse source
This commit is contained in:
Pierre-Yves Chibon 2017-07-13 17:52:32 +02:00
parent 2e51c4f77b
commit 51842f1648
2 changed files with 0 additions and 475 deletions
Download patch file Download diff file Expand all files Collapse all files

10
roles/distgit/tasks/main.yml
View file

@ -259,16 +259,6 @@
- config
- distgit
- name: install the genacls.pkgdb scripts
template: src={{item}} dest=/usr/local/bin/genacls.pkgdb
owner=root group=root mode=0755
with_items:
- genacls.pkgdb.stg
when: env == "staging"
tags:
- config
- distgit
- name: Add the genacl daily cron job
copy: src=genacls.cron dest=/etc/cron.d/genacls.cron
owner=root mode=644

465
roles/distgit/templates/genacls.pkgdb.stg
View file

@ -1,465 +0,0 @@
#!/usr/bin/python -t
#
# Create an /etc/gitolite/conf/gitolite.conf file with acls for dist-git
#
# Takes no arguments!
#
import copy
import grp
import itertools
import os
import sys
import json
from multiprocessing import Pool, Manager
import requests
from sqlalchemy.exc import SQLAlchemyError
TESTING = False
if 'PAGURE_CONFIG' not in os.environ \
and os.path.exists('/etc/pagure/pagure.cfg'):
if TESTING:
print 'Using configuration file `/etc/pagure/pagure.cfg`'
os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
import pagure
from pagure import SESSION
from pagure.exceptions import PagureException
{% if env == 'staging' %}
VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json'
GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json'
{% else %}
VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json'
GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json'
{% endif %}
RELENG_USER = {
'username': 'releng',
'fullname': 'Fedora Release Engineering',
'default_email': 'releng-team@fedoraproject.org',
}
def get_user_info(username):
''' Uses python-fedora to get information about a FAS user '''
user = {
'username': username,
'fullname': username,
'default_email': '%s@fedoraproject.org' % username
}
return user
def create_user_obj(session, username, userinfo=None):
''' Creates a sqlalchemy user object for pagure db '''
user = None
try:
if not userinfo:
userinfo = get_user_info(username)
user = pagure.lib.set_up_user(
session=session,
username=username,
fullname=userinfo['fullname'],
default_email=userinfo['default_email']
)
session.commit()
except SQLAlchemyError:
session.rollback()
if TESTING:
print 'Creating user failed'
return user
def create_groups_in_db(groups):
''' Creates groups in pagure db '''
group_keys = groups.keys()
for groupname in group_keys:
# we don't need to do anything with empty groups, do we?
if len(groups[groupname]) == 0:
continue
# first make sure that the users in the groups are present in the db
group_users = groups[groupname]
for guser in group_users:
user_obj = pagure.lib.search_user(SESSION, username=guser)
if not user_obj:
user_obj = create_user_obj(SESSION, guser)
# check if the groups are present in the db
group_obj = pagure.lib.search_groups(SESSION, group_name=groupname)
if not group_obj:
# add the group to the db using the first user in the group
try:
pagure.lib.add_group(
session=SESSION,
group_name=groupname,
display_name=groupname,
description=None,
group_type='user',
user='releng',
is_admin=False,
blacklist=pagure.APP.config['BLACKLISTED_GROUPS']
)
SESSION.commit()
except SQLAlchemyError:
SESSION.rollback()
if TESTING:
print 'Adding a user to group failed'
# now that all groups are present in the db
# make sure that all the members are there in the group in the db
for guser in group_users:
if not pagure.lib.is_group_member(SESSION, guser, groupname):
group_obj = pagure.lib.search_groups(
session=SESSION,
group_name=groupname
)
try:
pagure.lib.add_user_to_group(
session=SESSION,
username=guser,
group=group_obj,
user='releng',
is_admin=False
)
SESSION.commit()
except SQLAlchemyError:
SESSION.rollback()
if TESTING:
print 'Adding a user to group failed'
def update_owners_to_db(session, namespace, pkg, owners):
''' Adds owners to pagure db '''
pkg_obj = pagure.lib.get_project(
session, name=pkg, namespace=namespace)
for owner in owners:
# check if the owners are present in the db
# if not create them
owner_obj = pagure.lib.search_user(session, username=owner)
if not owner_obj:
owner_obj = create_user_obj(session, owner)
# this flag is for avoiding unnecessary db queries
created = False
if not pkg_obj:
try:
pagure.lib.new_project(
session=session,
user='releng',
namespace=namespace,
name=pkg,
blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
gitfolder=pagure.APP.config['GIT_FOLDER'],
docfolder=pagure.APP.config['DOCS_FOLDER'],
ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
ignore_existing_repo=True,
)
session.commit()
created = True
except SQLAlchemyError as err:
session.rollback()
if TESTING:
print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
except PagureException as err:
if TESTING:
print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
# so now the pkg surely exists, make the owner,
# the owner of the repo if he is not
if created:
pkg_obj = pagure.lib.get_project(
session=session,
name=pkg,
namespace=namespace
)
#if owner_obj not in pkg_obj.admins and owner_obj is not pkg_obj.user:
if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user:
try:
pagure.lib.add_user_to_project(
session=session,
project=pkg_obj,
new_user=owner_obj.user,
user='releng',
access='commit',
)
session.commit()
except SQLAlchemyError as err:
SESSION.rollback()
if TESTING:
print "Couldn't add user to project"
print "ERROR: %s" % err
def update_groups_to_db(session, namespace, pkg, pkg_groups):
''' Adds groups to projects in pagure db '''
pkg_obj = pagure.lib.get_project(
session, name=pkg, namespace=namespace)
for group in pkg_groups:
# we have already created all the groups
group_obj = pagure.lib.search_groups(session, group_name=group)
# in case when there are only groups with commit access and no
# people the flag is for cutting out db queries later
created = False
if not pkg_obj:
try:
pagure.lib.new_project(
session=session,
user='releng',
namespace=namespace,
name=pkg,
blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
gitfolder=pagure.APP.config['GIT_FOLDER'],
docfolder=pagure.APP.config['DOCS_FOLDER'],
ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
ignore_existing_repo=True,
)
session.commit()
created = True
except SQLAlchemyError as err:
session.rollback()
if TESTING:
print "Couldn't create project"
print "ERROR: %s" % err
except PagureException as err:
if TESTING:
print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
# for the case when the new project was just created
# by the above call
if created:
pkg_obj = pagure.lib.get_project(
session, name=pkg, namespace=namespace)
# if the group was initially empty, it was not
# created in the db
if not group_obj:
continue
# check if the group is added to project
# if not, add them
if group_obj not in pkg_obj.groups:
try:
pagure.lib.add_group_to_project(
session=session,
project=pkg_obj,
new_group=group,
user='releng',
access='commit',
)
session.commit()
except SQLAlchemyError as err:
session.rollback()
if TESTING:
print "Adding a group to a project failed"
print "ERROR: %s" % err
def add_fork_to_gitolite():
''' Creates a sqlalchemy user object for pagure db '''
for fork in pagure.lib.search_projects(session=SESSION, fork=True):
print ''
print 'repo %s' % (fork.fullname)
print ' RWC = %s' % fork.user.username
def process_pkg(arg):
""" Process the given package, adjust pagure for it and queue all the
ACLs so we can send them to gitolite
"""
pkg, acls, myq = arg
session = pagure.lib.create_session(pagure.APP.config['DB_URL'])
branchAcls = {} # Check whether we need to set separate per branch acls
buffer = [] # Buffer the output per package
masters = [] # Folks that have commit to master
writers = [] # Anybody that has write access
# Examine each branch in the package
branches = acls[pkg].keys()
branches.sort()
for branch in branches:
if branch not in ACTIVE.keys():
continue
if 'packager' in acls[pkg][branch]['commit']['groups']:
# If the packager group is defined, everyone has access
buffer.append(' RWC %s = @all' % (ACTIVE[branch]))
branchAcls.setdefault('@all', []).append(
(pkg, ACTIVE[branch])
)
if branch == 'master':
masters.append('@all')
if '@all' not in writers:
writers.append('@all')
else:
# Extract the owners
committers = []
owners = acls[pkg][branch]['commit']['people']
owners.sort()
for owner in owners:
committers.append(owner)
for group in acls[pkg][branch]['commit']['groups']:
committers.append('@%s' % group)
if branch == 'master':
masters.extend(committers)
pkg_groups = acls[pkg][branch]['commit']['groups']
update_owners_to_db(session, namespace, pkg, owners)
update_groups_to_db(session, namespace, pkg, pkg_groups)
# add all the committers to the top writers list
for committer in committers:
if committer not in writers:
writers.append(committer)
# Print the committers to the acl for this package-branch
committers = ' '.join(committers)
buffer.append(
' RWC %s = %s' % (ACTIVE[branch], committers))
branchAcls.setdefault(committers, []).append(
(pkg, ACTIVE[branch])
)
session.close()
data = [pkg, buffer, writers, masters]
myq.put(data)
myq.task_done()
if __name__ == '__main__':
# Create the rel-eng user if it needs to
if not pagure.lib.search_user(SESSION, username='releng'):
create_user_obj(SESSION, 'releng', RELENG_USER)
TRUSTED = grp.getgrnam('cvsadmin')[3]
ARM = grp.getgrnam('fedora-arm')[3]
SPARC = grp.getgrnam('fedora-sparc')[3]
IA64 = grp.getgrnam('fedora-ia64')[3]
S390 = grp.getgrnam('fedora-s390')[3]
PPC = grp.getgrnam('fedora-ppc')[3]
PROVEN = grp.getgrnam('provenpackager')[3]
# Set the active branches to create ACLs for
# Give them the git branch eqiv until pkgdb follows suite
ACTIVE = {
'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4',
'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6',
'epel7': 'epel7',
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24',
'f25': 'f25',
'devel': 'master', 'master': 'master'}
# Create a "regex"ish list 0f the reserved branches
RESERVED = [
'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]',
'olpc[0-9]']
# print out our user groups
print '@admins = %s' % ' '.join(TRUSTED)
print '@provenpackager = %s' % ' '.join(PROVEN)
print '@fedora-arm = %s' % ' '.join(ARM)
print '@fedora-s390 = %s' % ' '.join(S390)
print '@fedora-ppc = %s' % ' '.join(PPC)
groups = {
'admins': TRUSTED,
'fedora-arm': ARM,
'SPARC': SPARC,
'IA65': IA64,
'fedora-s390': S390,
'fedora-ppc': PPC,
'provenpackager': PROVEN
}
# Get a list of all the groups
groups_ = requests.get(GRP_URL).json()
for group in groups_['groups']:
print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3]))
gmems = grp.getgrnam(group)[3]
if group not in groups.keys():
groups[group] = gmems
elif groups[group] != gmems:
groups[group] = gmems
create_groups_in_db(groups)
# Check the blacklist and if the name clashes
# append '-1' after them - tmp workaround
#blacklist = pagure.APP.config.get('BLACKLISTED_PROJECTS')
#pkgs_list = data['rpms'].keys()
#for i in pkgs_list:
#if i in blacklist:
#data['rpms'][i + '-1'] = data['rpms'].pop(i)
data = requests.get(VCS_URL).json()
# Give a little space before moving onto the permissions
print ''
# print our default permissions
print 'repo @all'
print ' - VREF/update-block-push-origin = @all'
print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc'
print ' R = @all'
#print ' RW private- = @all'
# dont' enable the above until we prevent building for real from private-
# Get a list of all the packages
for namespace in data:
if namespace == 'title':
continue
acls = data[namespace]
pkglist = sorted(data[namespace].keys())
m = Manager()
q = m.Queue()
p = Pool(5)
p.map(process_pkg, itertools.product(pkglist, [acls], [q]))
p.close()
p.join()
#for pkg in pkglist:
#process_pkg([pkg, acls, q])
while q.qsize():
pkg, buffer, writers, masters = q.get()
print ''
print 'repo %s/%s' % (namespace, pkg)
print '\n'.join(buffer)
for reserved in RESERVED:
print ' - %s = @all' % reserved
print ' RWC refs/tags/ = %s' % ' '.join(writers)
if masters:
print ' RWC = %s' % ' '.join(masters)
q.join()
add_fork_to_gitolite()
sys.exit(0)