Merge branch 'master' of /git/ansible

inventory/group_vars/jenkins-slave

@@ -27,7 +27,6 @@ slave_packages_common:
 - python-nose
 - python-BeautifulSoup
 - python-fedora
-- python2-unittest2
 - python-pep8
 - python-psycopg2
 - postgresql-devel   # Required to install python-psycopg2 w/in a venv
@@ -45,13 +44,11 @@ slave_packages_common:
 - openssl-devel      # Required by bodhi/cffi/cryptography
 - redis              # Required by copr
 - createrepo_c       # Required by bodhi2
-- python2-createrepo_c       # Required by bodhi2
 - python-straight-plugin
 - pyflakes           # Requested by user rholy (ticket #4175)
 - koji               # Required by koschei (ticket #4852) and pyrpkg (ticket #4838)
 - rpm-python         # Required by koschei (ticket #4852)
 - libgit2-devel      # Required by pagure
-- python2-pygit2     # Required by pagure
 - osbs-client        # Required by pyrpkg (ticket #4838)
 - intltool           # Required by fedora-comps (ticket #5307)
 - fedpkg             # Required by fedora-kickstarts (ticket #5406)
@@ -161,7 +158,6 @@ slave_packages_fedora:
 - krb5-server
 - socket_wrapper
 - nss_wrapper
-- python-requests-kerberos
 - krb5-workstation
 - python-sssdconfig
 - mod_ssl
@@ -193,6 +189,9 @@ slave_packages_fedora:
 - rubygem-thor
 - rubygems
 - rubypick
+- python2-unittest2
+- python2-createrepo_c       # Required by bodhi2
+- python2-pygit2     # Required by pagure
 
 # Packages installed only on CentOS Jenkins slaves
 slave_packages_centos:
@@ -206,13 +205,18 @@ el7_only:
 - python-hawkey      # Required by koschei (ticket #4852)
 - python-librepo     # Required by koschei (ticket #4852)
 - nosync             # for use in mock
+- python-unittest2
+- python-createrepo_c       # Required by bodhi2
+- python-pygit2     # Required by pagure
 
 # Packages only available in f23+
 f23_only:
 - mod_auth_openidc
 - python2-solv       # For modulemd-resolver. (ticket #5349)
 - python3-solv       # For modulemd-resolver. (ticket #5349)
+- python-requests-kerberos
 
 # Packages only available/needed in f24+
 f24_only:
 - python2-systemd
+- python2-requests-kerberos

playbooks/groups/jenkins-slave.yml

@@ -28,6 +28,7 @@
 
 - name: provision instance
   hosts: jenkins-slave
+  gather_facts: True
   vars_files:
    - /srv/web/infra/ansible/vars/global.yml
    - "/srv/private/ansible/vars.yml"

roles/distgit/pagure/templates/pagure.cfg

@@ -174,6 +174,7 @@ ENABLE_NEW_PROJECTS = False
 ENABLE_DEL_PROJECTS = False
 ENABLE_TICKETS = False
 ENABLE_USER_MNGT = False
+ALLOWED_PREFIX = ['rpms', 'modules', 'docker']
 
 DISABLED_PLUGINS = ['IRC', 'Pagure tickets', 'Read the Doc']
 

roles/distgit/tasks/main.yml

@@ -186,6 +186,17 @@
             owner=root group=root mode=0755
   with_items:
     - genacls.pkgdb
+  when: env != "staging"
+  tags:
+  - config
+  - distgit
+
+- name: install the genacls.pkgdb scripts
+  template: src={{item}} dest=/usr/local/bin/genacls.pkgdb
+            owner=root group=root mode=0755
+  with_items:
+    - genacls.pkgdb.stg
+  when: env == "staging"
   tags:
   - config
   - distgit

roles/distgit/templates/genacls.pkgdb

@@ -38,7 +38,7 @@ if __name__ == '__main__':
         'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
         'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
         'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24',
-	'f25': 'f25',
+        'f25': 'f25',
         'devel': 'master', 'master': 'master'}
 
     # Create a "regex"ish list 0f the reserved branches

roles/distgit/templates/genacls.pkgdb.stg

@@ -0,0 +1,372 @@
+#!/usr/bin/python -t
+#
+# Create an /etc/gitolite/conf/gitolite.conf file with acls for dist-git
+#
+# Takes no arguments!
+#
+
+import copy
+import grp
+import sys
+import json
+
+import requests
+from sqlalchemy.exc import SQLAlchemyError
+
+
+if 'PAGURE_CONFIG' not in os.environ \
+        and os.path.exists('/etc/pagure/pagure.cfg'):
+    print 'Using configuration file `/etc/pagure/pagure.cfg`'
+    os.environ['PAGURE_CONFIG'] = '/etc/pagure/pagure.cfg'
+
+import pagure
+from pagure import SESSION
+
+TESTING = False
+
+{% if env == 'staging' %}
+VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json'
+GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json'
+{% else %}
+VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json'
+GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json'
+{% endif %}
+
+
+def create_user_obj(username):
+    ''' Creates a sqlalchemy user object for pagure db '''
+    try:
+        user = pagure.lib.set_up_user(
+            SESSION,
+            username,
+            username,
+            '%s@fedorahosted.org' % username
+        )
+        SESSION.commit()
+    except SQLAlchemyError:
+        SESSION.rollback()
+        if TESTING:
+            print 'Creating user failed'
+
+    return user
+
+
+def create_groups_in_db(groups):
+    ''' Creates groups in pagure db '''
+
+    group_keys = groups.keys()
+    for groupname in group_keys:
+
+        # we don't need to do anything with empty groups, do we?
+        if len(groups[groupname]) == 0:
+            continue
+
+        # first insure the users in the groups are present in the db
+        group_users = groups[groupname]
+        for guser in group_users:
+            user_obj = pagure.lib.search_user(SESSION, username=guser)
+            if not user_obj:
+                user_obj = create_user_obj(guser)
+
+        # check if the groups are present in the db
+        group_obj = pagure.lib.search_groups(SESSION, group_name=groupname)
+        if not group_obj:
+            # add the group to the db using the first user in the group
+            try:
+                pagure.lib.add_group(
+                    SESSION, groupname, 'user',
+                    groups[groupname][0], False,
+                    pagure.APP.config['BLACKLISTED_GROUPS'])
+                SESSION.commit()
+            except SQLAlchemyError:
+                SESSION.rollback()
+                if TESTING:
+                    print 'Adding a user to group failed'
+
+        # now that all groups are present in the db
+        # ensure all the members are there in the group in the db
+        for guser in group_users:
+            if not pagure.lib.is_group_member(SESSION, guser, groupname):
+                group_obj = pagure.lib.search_groups(
+                    SESSION, group_name=groupname)
+                try:
+                    msg = pagure.lib.add_user_to_group(
+                        SESSION,
+                        guser,
+                        group_obj,
+                        groups[groupname][0],
+                        False
+                    )
+                    SESSION.commit()
+                except SQLAlchemyError:
+                    SESSION.rollback()
+                    if TESTING:
+                        print 'Adding a user to group failed'
+
+
+def update_owners_to_db(pkg, owners):
+    ''' Adds owners to pagure db '''
+
+    for owner in owners:
+        # check if the owners are present in the db if not create them
+        owner_obj = pagure.lib.search_user(SESSION, username=owner)
+        if not owner_obj:
+            owner_obj = create_user_obj(owner)
+
+        # check if the repo exists, if not create
+        pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg)
+
+        # this flag is for avoiding unnecessary db queries
+        flag = True
+        if not pkg_obj:
+            pkgname = pkg
+            try:
+                msg = pagure.lib.new_project(
+                    SESSION,
+                    owner,
+                    'rpms/%s' % pkgname,
+                    pagure.APP.config['BLACKLISTED_PROJECTS'],
+                    pagure.APP.config['ALLOWED_PREFIX'],
+                    pagure.APP.config['GIT_FOLDER'],
+                    pagure.APP.config['DOCS_FOLDER'],
+                    pagure.APP.config['TICKETS_FOLDER'],
+                    pagure.APP.config['REQUESTS_FOLDER'],
+                )
+                SESSION.commit()
+                flag = False
+            except SQLAlchemyError:
+                SESSION.rollback()
+                if TESTING:
+                    print "Couldn't create project"
+
+        # so now the pkg surely exists, make the owner, the owner of the repo
+        # if (s)he is not
+        if not flag:
+            pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg)
+
+        if owner_obj not in pkg_obj.users and owner_obj is not pkg_obj.user:
+            try:
+                msg = pagure.lib.add_user_to_project(
+                    SESSION,
+                    pkg_obj,
+                    owner_obj.user,
+                    pkg_obj.user.user,
+                )
+                SESSION.commit()
+            except SQLAlchemyError:
+                SESSION.rollback()
+                if TESTING:
+                    print "Couldn't add user to project"
+
+
+def update_groups_to_db(pkg, pkg_groups):
+    ''' Adds groups to projects in pagure db '''
+
+    for group in pkg_groups:
+        # we have already created all the groups
+        group_obj = pagure.lib.search_groups(SESSION, group_name=group)
+        pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg)
+
+        # In case when there are only groups with commit access and no people
+        # The below flag is for cutting out db queries later
+        flag = True
+        if not pkg_obj:
+            pkgname = pkg
+            try:
+                msg = pagure.lib.new_project(
+                    SESSION,
+                    group_obj.creator.user,
+                    'rpms/%s' % pkgname,
+                    pagure.APP.config['BLACKLISTED_PROJECTS'],
+                    pagure.APP.config['ALLOWED_PREFIX'],
+                    pagure.APP.config['GIT_FOLDER'],
+                    pagure.APP.config['DOCS_FOLDER'],
+                    pagure.APP.config['TICKETS_FOLDER'],
+                    pagure.APP.config['REQUESTS_FOLDER'],
+                )
+                SESSION.commit()
+                flag = False
+            except SQLAlchemyError:
+                SESSION.rollback()
+                if TESTING:
+                    print "Couldn't create project"
+
+        # for the case when the new project was just created by the above call
+        if not flag:
+            pkg_obj = pagure.lib.get_project(SESSION, name='rpms/%s' % pkg)
+
+        # if the group was initially empty, it was not created in the db
+        if not group_obj:
+            continue
+
+        # check if the group is added to project if not, add them
+        if group_obj not in pkg_obj.groups:
+            group_creator = group_obj.creator
+            try:
+                pagure.lib.add_group_to_project(
+                    SESSION,
+                    pkg_obj,
+                    group,
+                    pkg_obj.user.user,
+                )
+                SESSION.commit()
+            except SQLAlchemyError as err:
+                SESSION.rollback()
+                if TESTING:
+                    print "Adding a group to a project failed"
+
+
+if __name__ == '__main__':
+
+    TRUSTED = grp.getgrnam('cvsadmin')[3]
+    ARM = grp.getgrnam('fedora-arm')[3]
+    SPARC = grp.getgrnam('fedora-sparc')[3]
+    IA64 = grp.getgrnam('fedora-ia64')[3]
+    S390 = grp.getgrnam('fedora-s390')[3]
+    PPC = grp.getgrnam('fedora-ppc')[3]
+    PROVEN = grp.getgrnam('provenpackager')[3]
+
+    # Set the active branches to create ACLs for
+    # Give them the git branch eqiv until pkgdb follows suite
+    ACTIVE = {
+        'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4',
+        'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6',
+        'epel7': 'epel7',
+        'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
+        'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
+        'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24',
+        'f25': 'f25',
+        'devel': 'master', 'master': 'master'}
+
+    # Create a "regex"ish list 0f the reserved branches
+    RESERVED = [
+        'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]',
+        'olpc[0-9]']
+
+    # print out our user groups
+    print '@admins = %s' % ' '.join(TRUSTED)
+    print '@provenpackager = %s' % ' '.join(PROVEN)
+    print '@fedora-arm = %s' % ' '.join(ARM)
+    print '@fedora-s390 = %s' % ' '.join(S390)
+    print '@fedora-ppc = %s' % ' '.join(PPC)
+
+    groups = {
+        'admins': TRUSTED,
+        'fedora-arm': ARM,
+        'SPARC': SPARC,
+        'IA65': IA64,
+        'fedora-s390': S390,
+        'fedora-ppc': PPC,
+        'provenpackager': PROVEN
+    }
+
+    # Get a list of all the groups
+    groups_ = requests.get(GRP_URL).json()
+    for group in groups_['groups']:
+        print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3]))
+        gmems = grp.getgrnam(group)[3]
+        if group not in groups.keys():
+            groups[group] = gmems
+        elif groups[group] != gmems:
+            groups[group] = gmems
+
+    # <pagure db create groups>
+    create_groups_in_db(groups)
+    # </pagure db>
+
+    data = requests.get(VCS_URL).json()
+
+    # Give a little space before moving onto the permissions
+    print ''
+    # print our default permissions
+    print 'repo @all'
+    print '    -   VREF/update-block-push-origin = @all'
+    print '    RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc'
+    print '    R = @all'
+    #print '    RW  private-     = @all'
+    # dont' enable the above until we prevent building for real from private-
+
+    # XXX - Insert an artificial namespace into the set of namespaces returned
+    # by pkgdb.  We want to create a mirror of rpms/PKG in rpms-checks/PKG
+    # This hack occurs in two places.  Here, and in the branch-creation script.
+    # https://github.com/fedora-infra/pkgdb2/issues/329#issuecomment-207050233
+    data['rpms-checks'] = copy.copy(data['rpms'])
+
+    # Get a list of all the packages
+    for key in data:
+        if key == 'title':
+            continue
+
+        acls = data[key]
+        pkglist = data[key].keys()
+        pkglist.sort()
+
+        if key != 'packageAcls':
+            key = '%s/' % key
+        else:
+            key = ''
+
+        for pkg in pkglist:
+
+            branchAcls = {}  # Check whether we need to set separate per branch acls
+            buffer = []  # Buffer the output per package
+            masters = []  # Folks that have commit to master
+            writers = []  # Anybody that has write access
+
+            # Examine each branch in the package
+            branches = acls[pkg].keys()
+            branches.sort()
+            for branch in branches:
+                if branch not in ACTIVE.keys():
+                    continue
+                if 'packager' in acls[pkg][branch]['commit']['groups']:
+                    # If the packager group is defined, everyone has access
+                    buffer.append('    RWC   %s = @all' % (ACTIVE[branch]))
+                    branchAcls.setdefault('@all', []).append(
+                        (pkg, ACTIVE[branch])
+                    )
+                    if branch == 'master':
+                        masters.append('@all')
+                    if '@all' not in writers:
+                        writers.append('@all')
+                else:
+                    # Extract the owners
+                    committers = []
+                    owners = acls[pkg][branch]['commit']['people']
+                    owners.sort()
+                    for owner in owners:
+                        committers.append(owner)
+                    for group in acls[pkg][branch]['commit']['groups']:
+                        committers.append('@%s' % group)
+                    if branch == 'master':
+                        masters.extend(committers)
+
+                        # <pagure db update groups and owner>
+                        pkg_groups = acls[pkg][branch]['commit']['groups']
+                        update_owners_to_db(pkg, owners)
+                        update_groups_to_db(pkg, pkg_groups)
+                        # </pagure db>
+
+                    # add all the committers to the top writers list
+                    for committer in committers:
+                        if committer not in writers:
+                            writers.append(committer)
+
+                    # Print the committers to the acl for this package-branch
+                    committers = ' '.join(committers)
+                    buffer.append(
+                        '    RWC   %s = %s' % (ACTIVE[branch], committers))
+                    branchAcls.setdefault(committers, []).append(
+                        (pkg, ACTIVE[branch])
+                    )
+
+            print ''
+            print 'repo %s%s' % (key, pkg)
+            print '\n'.join(buffer)
+            for reserved in RESERVED:
+                print '    -    %s = @all' % reserved
+            print '    RWC  refs/tags/ = %s' % ' '.join(writers)
+            if masters:
+                print '    RWC      = %s' % ' '.join(masters)
+
+    sys.exit(0)

roles/jenkins/slave/tasks/main.yml

@@ -19,16 +19,7 @@
 - name: install distro-specific slave packages (fedora, dnf)
   dnf: name={{ item }} state=installed
   with_items: "{{ slave_packages_fedora }}"
-  when: is_fedora is defined and ansible_distribution_major_version|int > 21
+  when: ansible_distribution_major_version|int > 21
-  tags:
-  - jenkins
-  - jenkins/slave
-  - packages
-
-- name: install distro-specific slave packages (fedora, yum)
-  yum: name={{ item }} state=installed
-  with_items: "{{ slave_packages_fedora }}"
-  when: is_fedora is defined and ansible_distribution_major_version|int < 22
   tags:
   - jenkins
   - jenkins/slave
@@ -37,7 +28,7 @@
 - name: install distro-specific slave packages (centos, yum)
   yum: name={{ item }} state=installed
   with_items: "{{ slave_packages_centos }}"
-  when: is_fedora is not defined
+  when: ansible_distribution_major_version|int < 8
   tags:
   - jenkins
   - jenkins/slave
@@ -52,19 +43,19 @@
   - jenkins/slave
   - packages
 
-- name: install f23 and higher specific slave packages
+- name: install f23 specific slave packages
   dnf: name={{ item }} state=installed
   with_items: "{{ f23_only }}"
-  when: ansible_distribution_major_version|int > 22
+  when: ansible_distribution_major_version|int == 23
   tags:
   - jenkins
   - jenkins/slave
   - packages
 
-- name: install f24 and higher specific slave packages
+- name: install f24 specific slave packages
   dnf: name={{ item }} state=installed
   with_items: "{{ f24_only }}"
-  when: ansible_distribution_major_version|int > 23
+  when: ansible_distribution_major_version|int == 24
   tags:
   - jenkins
   - jenkins/slave

roles/releng/files/aarch64.branched

@@ -1,3 +1,3 @@
 # branched devel compose
 MAILTO=releng-cron@lists.fedoraproject.org
-#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh arm
+15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh arm

roles/releng/files/power64.branched

@@ -1,3 +1,3 @@
 # branched devel compose
 MAILTO=releng-cron@lists.fedoraproject.org
-#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh ppc
+15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh ppc

roles/releng/files/s390.branched

@@ -1,3 +1,3 @@
 # branched devel compose
 MAILTO=releng-cron@lists.fedoraproject.org
-#15 10 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh s390
+15 10 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f25-secondary-arch && LANG=en_US.UTF-8 ./nightly.sh s390