Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Update the script syncing ACLs from pkgdb to dist-git and pagure to the recent changes made to pagure

Browse source
This commit is contained in:
Pierre-Yves Chibon 2017-02-13 10:34:59 +01:00
parent 8f32e86245
commit 5128d7e6f3
1 changed files with 126 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

210
roles/distgit/templates/genacls.pkgdb.stg
View file

@ -22,6 +22,7 @@ if 'PAGURE_CONFIG' not in os.environ \
import pagure import pagure
from pagure import SESSION from pagure import SESSION
from pagure.exceptions import PagureException
TESTING = False TESTING = False
@ -34,14 +35,26 @@ GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json'
{% endif %} {% endif %}
def get_user_info(username):
''' Uses python-fedora to get information about a FAS user '''
user = {
'username': username,
'fullname': username,
'default_email': '%s@fedoraproject.org' % username
}
return user
def create_user_obj(username): def create_user_obj(username):
''' Creates a sqlalchemy user object for pagure db ''' ''' Creates a sqlalchemy user object for pagure db '''
try: try:
userinfo = get_user_info(username)
user = pagure.lib.set_up_user( user = pagure.lib.set_up_user(
SESSION, session=SESSION,
username, username=username,
username, fullname=userinfo['fullname'],
'%s@fedorahosted.org' % username default_email=userinfo['default_email']
) )
SESSION.commit() SESSION.commit()
except SQLAlchemyError: except SQLAlchemyError:
@ -62,7 +75,7 @@ def create_groups_in_db(groups):
if len(groups[groupname]) == 0: if len(groups[groupname]) == 0:
continue continue
# first insure the users in the groups are present in the db # first make sure that the users in the groups are present in the db
group_users = groups[groupname] group_users = groups[groupname]
for guser in group_users: for guser in group_users:
user_obj = pagure.lib.search_user(SESSION, username=guser) user_obj = pagure.lib.search_user(SESSION, username=guser)
@ -75,9 +88,15 @@ def create_groups_in_db(groups):
# add the group to the db using the first user in the group # add the group to the db using the first user in the group
try: try:
pagure.lib.add_group( pagure.lib.add_group(
SESSION, groupname, 'user', session=SESSION,
groups[groupname][0], False, group_name=groupname,
pagure.APP.config['BLACKLISTED_GROUPS']) display_name=groupname,
description=None,
group_type='user',
user=groups[groupname][0],
is_admin=False,
blacklist=pagure.APP.config['BLACKLISTED_GROUPS']
)
SESSION.commit() SESSION.commit()
except SQLAlchemyError: except SQLAlchemyError:
SESSION.rollback() SESSION.rollback()
@ -85,18 +104,20 @@ def create_groups_in_db(groups):
print 'Adding a user to group failed' print 'Adding a user to group failed'
# now that all groups are present in the db # now that all groups are present in the db
# ensure all the members are there in the group in the db # make sure that all the members are there in the group in the db
for guser in group_users: for guser in group_users:
if not pagure.lib.is_group_member(SESSION, guser, groupname): if not pagure.lib.is_group_member(SESSION, guser, groupname):
group_obj = pagure.lib.search_groups( group_obj = pagure.lib.search_groups(
SESSION, group_name=groupname) session=SESSION,
group_name=groupname
)
try: try:
msg = pagure.lib.add_user_to_group( pagure.lib.add_user_to_group(
SESSION, session=SESSION,
guser, username=guser,
group_obj, group=group_obj,
groups[groupname][0], user=groups[groupname][0],
False is_admin=False
) )
SESSION.commit() SESSION.commit()
except SQLAlchemyError: except SQLAlchemyError:
@ -105,116 +126,139 @@ def create_groups_in_db(groups):
print 'Adding a user to group failed' print 'Adding a user to group failed'
def update_owners_to_db(pkg, owners): def update_owners_to_db(namespace, pkg, owners):
''' Adds owners to pagure db ''' ''' Adds owners to pagure db '''
for owner in owners: for owner in owners:
# check if the owners are present in the db if not create them # check if the owners are present in the db
# if not create them
owner_obj = pagure.lib.search_user(SESSION, username=owner) owner_obj = pagure.lib.search_user(SESSION, username=owner)
if not owner_obj: if not owner_obj:
owner_obj = create_user_obj(owner) owner_obj = create_user_obj(owner)
# check if the repo exists, if not create pkg_obj = pagure.lib.get_project(
pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) SESSION, name=pkg, namespace=namespace)
# this flag is for avoiding unnecessary db queries # this flag is for avoiding unnecessary db queries
flag = True flag = True
if not pkg_obj: if not pkg_obj:
pkgname = pkg
try: try:
msg = pagure.lib.new_project( pagure.lib.new_project(
SESSION, session=SESSION,
owner, user=owner,
'rpms/%s' % pkgname, namespace=namespace,
pagure.APP.config['BLACKLISTED_PROJECTS'], name=pkg,
pagure.APP.config['ALLOWED_PREFIX'], blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
pagure.APP.config['GIT_FOLDER'], allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
pagure.APP.config['DOCS_FOLDER'], gitfolder=pagure.APP.config['GIT_FOLDER'],
pagure.APP.config['TICKETS_FOLDER'], docfolder=pagure.APP.config['DOCS_FOLDER'],
pagure.APP.config['REQUESTS_FOLDER'], ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
) )
SESSION.commit() SESSION.commit()
flag = False flag = False
except SQLAlchemyError: except SQLAlchemyError as err:
SESSION.rollback() SESSION.rollback()
if TESTING: if TESTING:
print "Couldn't create project" print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
except PagureException as err:
if TESTING:
print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
# so now the pkg surely exists, make the owner, the owner of the repo # so now the pkg surely exists, make the owner,
# if (s)he is not # the owner of the repo if he is not
if not flag: if not flag:
pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) pkg_obj = pagure.lib.get_project(
session=SESSION,
name=pkg,
namespace=namespace
)
if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user: if owner_obj not in pkg_obj.admins and owner_obj is not pkg_obj.user:
try: try:
msg = pagure.lib.add_user_to_project( pagure.lib.add_user_to_project(
SESSION, session=SESSION,
pkg_obj, project=pkg_obj,
owner_obj.user, new_user=owner_obj.user,
pkg_obj.user.user, user=pkg_obj.user.user,
) )
SESSION.commit() SESSION.commit()
except SQLAlchemyError: except SQLAlchemyError as err:
SESSION.rollback() SESSION.rollback()
if TESTING: if TESTING:
print "Couldn't add user to project" print "Couldn't add user to project"
print "ERROR: %s" % err
def update_groups_to_db(pkg, pkg_groups): def update_groups_to_db(namespace, pkg, pkg_groups):
''' Adds groups to projects in pagure db ''' ''' Adds groups to projects in pagure db '''
for group in pkg_groups: for group in pkg_groups:
# we have already created all the groups # we have already created all the groups
group_obj = pagure.lib.search_groups(SESSION, group_name=group) group_obj = pagure.lib.search_groups(SESSION, group_name=group)
pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg)
# In case when there are only groups with commit access and no people pkg_obj = pagure.lib.get_project(
# The below flag is for cutting out db queries later SESSION, name=pkg, namespace=namespace)
# in case when there are only groups with commit access and no
# people the flag is for cutting out db queries later
flag = True flag = True
if not pkg_obj: if not pkg_obj:
pkgname = pkg
try: try:
msg = pagure.lib.new_project( pagure.lib.new_project(
SESSION, session=SESSION,
group_obj.creator.user, user=group_obj.creator.user,
'rpms/%s' % pkgname, namespace=namespace,
pagure.APP.config['BLACKLISTED_PROJECTS'], name=pkg,
pagure.APP.config['ALLOWED_PREFIX'], blacklist=pagure.APP.config['BLACKLISTED_PROJECTS'],
pagure.APP.config['GIT_FOLDER'], allowed_prefix=pagure.APP.config['ALLOWED_PREFIX'],
pagure.APP.config['DOCS_FOLDER'], gitfolder=pagure.APP.config['GIT_FOLDER'],
pagure.APP.config['TICKETS_FOLDER'], docfolder=pagure.APP.config['DOCS_FOLDER'],
pagure.APP.config['REQUESTS_FOLDER'], ticketfolder=pagure.APP.config['TICKETS_FOLDER'],
requestfolder=pagure.APP.config['REQUESTS_FOLDER'],
) )
SESSION.commit() SESSION.commit()
flag = False flag = False
except SQLAlchemyError: except SQLAlchemyError as err:
SESSION.rollback() SESSION.rollback()
if TESTING: if TESTING:
print "Couldn't create project" print "Couldn't create project"
print "ERROR: %s" % err
except PagureException as err:
if TESTING:
print "Couldn't create project - %s" % pkg
print "ERROR: %s" % err
# for the case when the new project was just created by the above call # for the case when the new project was just created
# by the above call
if not flag: if not flag:
pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg) pkg_obj = pagure.lib.get_project(
SESSION, name=pkg, namespace=namespace)
# if the group was initially empty, it was not created in the db # if the group was initially empty, it was not
# created in the db
if not group_obj: if not group_obj:
continue continue
# check if the group is added to project if not, add them # check if the group is added to project
# if not, add them
if group_obj not in pkg_obj.groups: if group_obj not in pkg_obj.groups:
group_creator = group_obj.creator
try: try:
pagure.lib.add_group_to_project( pagure.lib.add_group_to_project(
SESSION, session=SESSION,
pkg_obj, project=pkg_obj,
group, new_group=group,
pkg_obj.user.user, user=pkg_obj.user.user,
access='admin'
) )
SESSION.commit() SESSION.commit()
except SQLAlchemyError as err: except SQLAlchemyError as err:
SESSION.rollback() SESSION.rollback()
if TESTING: if TESTING:
print "Adding a group to a project failed" print "Adding a group to a project failed"
print "ERROR: %s" % err
if __name__ == '__main__': if __name__ == '__main__':
@ -271,9 +315,15 @@ if __name__ == '__main__':
elif groups[group] != gmems: elif groups[group] != gmems:
groups[group] = gmems groups[group] = gmems
# <pagure db create groups>
create_groups_in_db(groups) create_groups_in_db(groups)
# </pagure db>
# Check the blacklist and if the name clashes
# append '-1' after them - tmp workaround
#blacklist = pagure.APP.config.get('BLACKLISTED_PROJECTS')
#pkgs_list = data['rpms'].keys()
#for i in pkgs_list:
#if i in blacklist:
#data['rpms'][i + '-1'] = data['rpms'].pop(i)
data = requests.get(VCS_URL).json() data = requests.get(VCS_URL).json()
@ -303,18 +353,12 @@ if __name__ == '__main__':
data['test-docker'] = copy.copy(data['docker']) data['test-docker'] = copy.copy(data['docker'])
# Get a list of all the packages # Get a list of all the packages
for key in data: for namespace in data:
if key == 'title': if namespace == 'title':
continue continue
acls = data[key] acls = data[namespace]
pkglist = data[key].keys() pkglist = sorted(data[namespace].keys())
pkglist.sort()
if key != 'packageAcls':
key = '%s/' % key
else:
key = ''
for pkg in pkglist: for pkg in pkglist:
@ -351,11 +395,9 @@ if __name__ == '__main__':
if branch == 'master': if branch == 'master':
masters.extend(committers) masters.extend(committers)
# <pagure db update groups and owner>
pkg_groups = acls[pkg][branch]['commit']['groups'] pkg_groups = acls[pkg][branch]['commit']['groups']
update_owners_to_db(pkg, owners) update_owners_to_db(namespace, pkg, owners)
update_groups_to_db(pkg, pkg_groups) update_groups_to_db(namespace, pkg, pkg_groups)
# </pagure db>
# add all the committers to the top writers list # add all the committers to the top writers list
for committer in committers: for committer in committers: