Infrastructure/ansible
5
0
Fork 0
Code Issues Pull requests 3 Projects Releases Packages Wiki Activity Actions

Nuke pkgdb some more... and probably break everything.

Browse source 
Signed-off-by: Rick Elrod <relrod@redhat.com>
This commit is contained in:
Rick Elrod 2018-07-19 20:36:54 +00:00
parent 0e04af843a
commit 4a60ddc875
69 changed files with 22 additions and 3248 deletions
Download patch file Download diff file Expand all files Collapse all files

49
inventory/group_vars/pkgdb
View file

@ -1,49 +0,0 @@
---
# Define resources for this group of hosts here.
lvm_size: 20000
mem_size: 4096
num_cpus: 2
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
wsgi_fedmsg_service: pkgdb2
wsgi_procs: 3
wsgi_threads: 2
tcp_ports: [ 80 ]
fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-releng,sysadmin-cvs,sysadmin-veteran
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: shell
owner: root
group: sysadmin
can_send:
- logger.log
- service: pkgdb
owner: root
group: apache
alias: pkgdb2
can_send:
- pkgdb.acl.delete
- pkgdb.acl.update
- pkgdb.admin.action.status.update
- pkgdb.branch.complete
- pkgdb.branch.start
- pkgdb.collection.new
- pkgdb.collection.update
- pkgdb.owner.update
- pkgdb.package.branch.delete
- pkgdb.package.branch.new
- pkgdb.package.branch.request
- pkgdb.package.critpath.update
- pkgdb.package.delete
- pkgdb.package.monitor.update
- pkgdb.package.koschei.update
- pkgdb.package.new
- pkgdb.package.new.request
- pkgdb.package.unretire.request
- pkgdb.package.update
- pkgdb.package.update.status

49
inventory/group_vars/pkgdb-stg
View file

@ -1,49 +0,0 @@
---
# Define resources for this group of hosts here.
lvm_size: 20000
mem_size: 4096
num_cpus: 2
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
wsgi_fedmsg_service: pkgdb2
wsgi_procs: 2
wsgi_threads: 2
tcp_ports: [ 80 ]
fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-releng,sysadmin-cvs,sysadmin-veteran
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: shell
owner: root
group: sysadmin
can_send:
- logger.log
- service: pkgdb
owner: root
group: apache
alias: pkgdb2
can_send:
- pkgdb.acl.delete
- pkgdb.acl.update
- pkgdb.admin.action.status.update
- pkgdb.branch.complete
- pkgdb.branch.start
- pkgdb.collection.new
- pkgdb.collection.update
- pkgdb.owner.update
- pkgdb.package.branch.delete
- pkgdb.package.branch.new
- pkgdb.package.branch.request
- pkgdb.package.critpath.update
- pkgdb.package.delete
- pkgdb.package.monitor.update
- pkgdb.package.koschei.update
- pkgdb.package.new
- pkgdb.package.new.request
- pkgdb.package.unretire.request
- pkgdb.package.update
- pkgdb.package.update.status

2
inventory/group_vars/pkgs
View file

@ -58,8 +58,6 @@ fedmsg_certs:
- git.branch
- git.mass_branch.complete
- git.mass_branch.start
- git.pkgdb2branch.complete
- git.pkgdb2branch.start
- logger.log
- pagure.git.receive
- service: scm

4
inventory/group_vars/pkgs-stg
View file

@ -55,8 +55,6 @@ fedmsg_certs:
- git.branch
- git.mass_branch.complete
- git.mass_branch.start
- git.pkgdb2branch.complete
- git.pkgdb2branch.start
- pagure.git.receive
- service: scm
owner: root
@ -65,8 +63,6 @@ fedmsg_certs:
- git.branch
- git.mass_branch.complete
- git.mass_branch.start
- git.pkgdb2branch.complete
- git.pkgdb2branch.start
- git.receive
- pagure.git.receive
- service: lookaside

2
inventory/host_vars/db01.phx2.fedoraproject.org
View file

@ -30,7 +30,6 @@ databases:
- nuancier_lite
- odcs
- pdc
- pkgdb2
- statscache
- tahrir
- waiverdb
@ -55,7 +54,6 @@ dbs_to_backup:
- nuancier_lite
- odcs
- pdc
- pkgdb2
- statscache
- tahrir
- waiverdb

1
inventory/host_vars/db01.stg.phx2.fedoraproject.org
View file

@ -23,7 +23,6 @@ databases:
- mirrormanager
- notifications
- nuancier_lite
- pkgdb2
- tahrir
# These are normally group variables, but in this case db servers are often different

9
inventory/inventory
View file

@ -625,13 +625,6 @@ packages04.phx2.fedoraproject.org
[packages-stg]
packages03.stg.phx2.fedoraproject.org
[pkgdb]
pkgdb01.phx2.fedoraproject.org
pkgdb02.phx2.fedoraproject.org
[pkgdb-stg]
pkgdb01.stg.phx2.fedoraproject.org
[pkgs]
pkgs02.phx2.fedoraproject.org
@ -805,7 +798,6 @@ pdc-backend03.stg.phx2.fedoraproject.org
pdc-web01.stg.phx2.fedoraproject.org
pgbdr01.stg.phx2.fedoraproject.org
pgbdr02.stg.phx2.fedoraproject.org
pkgdb01.stg.phx2.fedoraproject.org
pkgs01.stg.phx2.fedoraproject.org
proxy01.stg.phx2.fedoraproject.org
qa-stg01.qa.fedoraproject.org ansible_port=222
@ -1243,7 +1235,6 @@ download
bvirthost
packages
# not yet created
#pkgdb
koji
dbserver

1
playbooks/fedmsgupdate.yml
View file

@ -20,7 +20,6 @@
- include_playbook: /srv/web/infra/ansible/playbooks/groups/notifs-web.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/nuancier.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/packages.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/pkgdb.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/releng.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/tagger.yml
- include_playbook: /srv/web/infra/ansible/playbooks/groups/wiki.yml

4
playbooks/groups/bodhi-backend.yml
View file

@ -52,10 +52,6 @@
process: fedmsg-hub
user: masher
- role: keytab/service
service: pkgdb
owner_user: fedmsg
owner_group: fedmsg
- role: keytab/service
owner_user: apache
owner_group: apache

67
playbooks/groups/pkgdb.yml
View file

@ -1,67 +0,0 @@
# create a new pkgdb server
# NOTE: should be used with --limit most of the time
# NOTE: make sure there is room/space for this server on the vmhost
# NOTE: most of these vars_path come from group_vars/pkgdb* or from hostvars
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pkgdb:pkgdb-stg"
- name: make the box be real
hosts: pkgdb-stg:pkgdb
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- base
- rkhunter
- nagios_client
- hosts
- fas_client
- collectd/base
- sudo
- { role: openvpn/client,
when: env != "staging" }
- mod_wsgi
tasks:
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
- import_tasks: "{{ tasks_path }}/2fa_client.yml"
- import_tasks: "{{ tasks_path }}/motd.yml"
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: set up fedmsg on pkgdb
hosts: pkgdb-stg:pkgdb
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- "{{ vars_path }}/{{ ansible_distribution }}.yml"
roles:
- fedmsg/base
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: deploy pkgdb itself
hosts: pkgdb-stg:pkgdb
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- "{{ vars_path }}/{{ ansible_distribution }}.yml"
roles:
- pkgdb2
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"

2
playbooks/groups/sundries.yml
View file

@ -27,8 +27,6 @@
- geoip-city-wsgi/app
- role: easyfix/gather
when: master_sundries_node
- role: fedora_owner_change
when: master_sundries_node and env != "staging"
- role: bz_review_report
when: master_sundries_node and env != "staging"
- rsyncd

19
playbooks/manual/restart-pkgdb.yml
View file

@ -1,19 +0,0 @@
- name: verify the frontend and stop it
hosts: pkgdb:pkgdb-stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
tasks:
- name: Start apache
service: name="httpd" state=started
post_tasks:
- name: tell nagios to unshush w.r.t. apache
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true

74
playbooks/manual/staging-sync/scripts/create-pkgdb-stg-modules.py
View file

@ -1,74 +0,0 @@
#!/usr/bin/env python
""" Create modules/ pkgdb entries in staging.
"""
import argparse
import sys
import pkgdb2client
parser = argparse.ArgumentParser(description='Create new modules in stg pkgdb')
parser.add_argument('--users',
help='A comma separated list of users '
'to get acls on the new modules.')
parser.add_argument('--modules',
help='A comma separated list of module '
'names to be created in staging.')
args = parser.parse_args()
users = [user.strip() for user in args.users.split(',')]
modules = [module.strip() for module in args.modules.split(',')]
print "%r will get full ACLs on new modules %r" % (users, modules)
response = raw_input("Does that look correct? [y/N]")
if response.lower() != 'y':
print "Exiting."
sys.exit(0)
else:
print "Ok."
client = pkgdb2client.PkgDB(
'https://admin.stg.fedoraproject.org/pkgdb',
login_callback=pkgdb2client.ask_password,
)
modules = [ {
'name': name,
'summary': 'The %s module' % name,
'description': 'This is a test entry for modularity development.',
'review_url': 'https://fedoraproject.org/wiki/Modularity',
'upstream_url': 'https://fedoraproject.org/wiki/Modularity',
'status': 'Approved',
'namespace': 'modules',
} for name in modules]
for module in modules:
print "Handling %s/%s" % (module['namespace'], module['name'])
client.create_package(
pkgname=module['name'],
summary=module['summary'],
description=module['description'],
review_url=module['review_url'],
status=module['status'],
shouldopen='whatever', # unused..
branches='master',
poc=users[0],
upstream_url=module['upstream_url'],
namespace=module['namespace'],
critpath=False,
#monitoring_status=False,
#koschei=False
)
for user in users:
print " Granting all to %r" % user
client.update_acl(
pkgname=module['name'],
namespace=module['namespace'],
branches='master',
acls=['watchcommits', 'watchbugzilla', 'approveacls', 'commit'],
status='Approved',
user=user,
)

21
playbooks/manual/staging-sync/scripts/export-pkgdb-stg-modules.py
View file

@ -1,21 +0,0 @@
#!/usr/bin/env python
""" Save the staging modules/ pkgdb entries to a local json file.
Use this script to save pkgdb modules/ entries in staging before wiping the db.
"""
import json
import requests
filename = 'old-modules.json'
response = requests.get(
'https://admin.stg.fedoraproject.org/pkgdb/api/packages/',
params=dict(namespace='modules'),
)
data = response.json()
with open(filename, 'wb') as f:
f.write(json.dumps(data, indent=2).encode('utf-8'))
print "Wrote %s" % filename

60
playbooks/manual/staging-sync/scripts/restore-pkgdb-stg-modules.py
View file

@ -1,60 +0,0 @@
#!/usr/bin/env python
""" Restore modules/ pkgdb entries in staging.
Use this script to restore pkgdb entries in staging after having wiped that db.
The workflow usually goes:
- Run the `export-pkgdb-stg-modules.py` script. This will save the modules and
their acls out to a local `old-modules.json` file.
- Run the `playbooks/manual/staging-sync/db-sync.yml` playbook. This will nuke
the staging pkgdb db and replace it with a copy of the current production
pkgdb db.
- Run this `restore-pkgdb-stg-modules.py` script. It will read in that
`old-modules.json` script and then recreate those entries in the new fresh
staging db.
"""
import json
import pkgdb2client
client = pkgdb2client.PkgDB(
'https://admin.stg.fedoraproject.org/pkgdb',
login_callback=pkgdb2client.ask_password,
)
with open('old-modules.json', 'rb') as f:
data = json.loads(f.read().decode('utf-8'))
for package in data['packages']:
print "Handling %s/%s" % (package['namespace'], package['name'])
client.create_package(
pkgname=package['name'],
summary=package['summary'],
description=package['description'],
review_url=package['review_url'],
status=package['status'],
shouldopen='whatever', # unused..
branches='master',
poc=package['acls'][0]['point_of_contact'],
upstream_url=package['upstream_url'],
namespace=package['namespace'],
critpath=False,
#monitoring_status=False,
#koschei=False
)
users = set([i['fas_name'] for i in package['acls'][0]['acls']])
for user in users:
print " Granting all to %r" % user
client.update_acl(
pkgname=package['name'],
namespace=package['namespace'],
branches='master',
acls=['watchcommits', 'watchbugzilla', 'approveacls', 'commit'],
status='Approved',
user=user,
)

19
playbooks/manual/stop_pkgdb.yml
View file

@ -1,19 +0,0 @@
- name: verify the frontend and stop it
hosts: pkgdb:pkgdb-stg
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
pre_tasks:
- name: tell nagios to shush w.r.t. apache
nagios: action=downtime minutes=90 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true
tasks:
- name: Stop apache
service: name="httpd" state=stopped

16
playbooks/run_pkgdb_sync_git.yml
View file

@ -1,16 +0,0 @@
# Run `pkgdb2branch` on
#
#To update from testing, adjust as follow:
# --extra-vars="package='pkg1:pkg2:pkg3'"
- name: run pkgdb_sync_git_branches.py
hosts: pkgs01.phx2.fedoraproject.org
user: root
serial: 25
gather_facts: False
tasks:
- name: call pkgdb_sync_git_branches.py
command: /usr/local/bin/pkgdb_sync_git_branches.py

13
roles/apps-fp-o/files/apps.yaml
View file

@ -437,19 +437,6 @@ children:
description: >
Copr is an easy-to-use automatic build system providing a
package repository as its output. You can make your **own** repositories!
- name: PkgDB
data:
url: https://admin.fedoraproject.org/pkgdb
user_url: https://admin.fedoraproject.org/pkgdb/packager/{user}/
package_url: https://admin.fedoraproject.org/pkgdb/package/{package}/
source_url: https://github.com/fedora-infra/pkgdb2
bugs_url: https://github.com/fedora-infra/pkgdb2/issues
docs_url: https://pkgdb2.readthedocs.org/en/latest/
sops:
- https://infrastructure.fedoraproject.org/infra/docs/packagedatabase.rst
status_mappings: ['pkgdb']
description: >
Manage ACLs of your packages.
- name: Koji
data:
icon: koji.png

7
roles/badges/backend/templates/badges-awarder.py
View file

@ -42,12 +42,7 @@ config = {
},
# Stuff used for caching packagedb relations.
"fedbadges.rules.utils.use_pkgdb2": True,
{% if env == 'staging' %}
"fedbadges.rules.utils.pkgdb_url": "https://admin.stg.fedoraproject.org/pkgdb/api",
{% else %}
"fedbadges.rules.utils.pkgdb_url": "https://admin.fedoraproject.org/pkgdb/api",
{% endif %}
"fedbadges.rules.utils.use_pkgdb2": False,
"fedbadges.rules.cache": {
"backend": "dogpile.cache.dbm",
"expiration_time": 300,

211
roles/bodhi2/backend/templates/owner-sync-pkgdb.j2
View file

@ -1,211 +0,0 @@
#!/usr/bin/python2
# cronjobs are run on releng01.stg
# Looks like:
# /usr/local/bin/owner-sync-pkgdb f19
# /usr/local/bin/owner-sync-pkgdb dist-5E-epel
# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
# /usr/local/bin/owner-sync-pkgdb epel7
import sys
import os
import ConfigParser
from urlparse import urljoin
import requests
DEBUG=False
VERIFY=True
{% if env == 'staging' %}
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.stg.fedoraproject.org/pkgdb/'
{% else %}
BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/'
{% endif %}
if not BASEURL.endswith('/'):
BASEURL = BASEURL + '/'
# Why do we have this? Seems insecure....
sys.path.append('.')
import koji
rawhide = '27'
extraArchList = {'kernel': ('i586', 'i686', 'noarch'),
'kernel-xen-2.6': ('i586', 'i686', 'noarch'),
'glibc': ('i686',),
'openssl': ('i686',),
'em8300-kmod': ('i586', 'i686'),
'sysprof-kmod': ('i586', 'i686'),
}
def usage():
print "Usage: owner-sync <tag>"
print " <tag>: tag to synchronize owners for"
sys.exit(1)
def get_options():
# shamelessly stolen from koji CLI
opts = {
{% if env == 'staging' %}
'server': 'https://koji.stg.fedoraproject.org/kojihub',
'weburl': 'https://koji.stg.fedoraproject.org/koji',
{% else %}
'server': 'https://koji.fedoraproject.org/kojihub',
'weburl': 'https://koji.fedoraproject.org/koji',
{% endif %}
'principal': 'pkgdb/{{inventory_hostname}}@{{ipa_realm}}',
'keytab': '/etc/krb5.pkgdb_{{inventory_hostname}}.keytab',
}
for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
if os.access(configFile, os.F_OK):
f = open(configFile)
config = ConfigParser.ConfigParser()
config.readfp(f)
f.close()
if config.has_section('koji'):
for name, value in config.items('koji'):
if opts.has_key(name):
opts[name] = value
for entry in opts.keys():
if entry == 'server' or entry == 'weburl':
pass
return opts
if __name__ == '__main__':
try:
tag=sys.argv[1]
except:
print "ERROR: no tag specified!\n"
usage()
if 'container' in tag:
namespace='container'
version = tag.split('-')[0].split('f')[1]
elif tag == 'module-package-list':
# See https://pagure.io/releng/issue/6663
# and https://pagure.io/fm-orchestrator/issue/333
namespace = 'rpms'
version = rawhide
else:
namespace='rpms'
if tag.startswith('epel'):
version = tag.split('epel')[1]
elif tag.startswith('f'):
version = tag.split('f')[1]
else:
version = tag.split('-')[1][:-1]
if tag.startswith('epel'):
# Ex: epel7 => epel7
reponame = tag
arches = ["primary"]
elif tag.endswith('epel'):
# Ex: dist-6E-epel => el6
reponame = 'el%s' % version
arches = ["primary"]
elif tag == 'module-package-list':
reponame = 'master'
arches = ["primary"]
else:
# Fedora
if version == rawhide:
reponame = 'master'
else:
reponame = tag.split('-')[0]
{% if env == 'staging' %}
arches = ["primary"]
{% else %}
if version <= "25":
arches = ["primary", "arm", "ppc", "s390"]
else:
arches = ["primary", "s390"]
{% endif %}
data = requests.get(urljoin(BASEURL, 'api/collections'), verify=VERIFY).json()
branch_names = set()
for collection in (c for c in data['collections'] if c['status'] != 'EOL'):
### TODO: check with pingou that this is now returning the same
# format as the collection names in api/vcs
# By moving the data from gitbranchname into branchname, I think
# that the data will now match
branch_names.add(collection['branchname'])
if reponame not in branch_names:
print 'tag %s => repo %s: does not seem to be a non-EOL branch' % (tag, reponame)
sys.exit(1)
data = requests.get(urljoin(BASEURL, 'api/vcs?format=json'), verify=VERIFY).json()
acls = data[namespace]
pkgs = {}
for pkg_name in acls:
try:
owners = acls[pkg_name][reponame]
except KeyError:
# Package is not branched for this release
continue
if len(owners['commit']['people']):
# Arbitrarily take the first committer listed as the owner in
# koji
pkgs[pkg_name] = owners['commit']['people'][0]
else:
pkgs[pkg_name] = 'orphan'
if tag == 'module-package-list' and not 'module-build-macros' in pkgs:
pkgs['module-build-macros'] = 'releng'
pkgList = pkgs.keys()
pkgList.sort()
options = get_options()
for arch in arches:
if arch == "primary":
session = koji.ClientSession("https://koji{{env_suffix}}.fedoraproject.org/kojihub", {'krb_rdns': False})
else:
session = koji.ClientSession("https://%s.koji.fedoraproject.org/kojihub" % arch, {'krb_rdns': False})
try:
session.krb_login(options['principal'], options['keytab'])
except:
print "Unable to sync to %s hub" % arch
continue
kojitag = session.getTag(tag)
if kojitag is None:
print "ERROR: tag %s does not exist for arch %s!\n" % (tag, arch)
usage()
kojipkgs = {}
kojiusers = [user['name'] for user in session.listUsers()]
for p in session.listPackages(tagID=tag, inherited = True):
kojipkgs[p['package_name']] = p
for pkg in pkgList:
owner = pkgs[pkg]
if DEBUG:
print '[DEBUG] Package: %s, Owner: %s' % (pkg, owner)
if not owner in kojiusers:
# add the user first
if DEBUG:
print "Adding user %s" % owner
else:
try:
session.createUser(owner)
except:
# user already exists
continue
kojiusers.append(owner)
if not kojipkgs.has_key(pkg):
if DEBUG:
print "Adding package %s for %s with owner %s" % (pkg, tag, owner)
else:
extraArches = None
if pkg in extraArchList:
extraArches = extraArchList[pkg]
session.packageListAdd(tag, pkg, owner = owner, extra_arches=extraArches)
elif kojipkgs[pkg]['owner_name'] != owner:
if DEBUG:
print "Setting owner for %s in %s to %s" % (pkg, tag, owner)
else:
session.packageListSetOwner(tag, pkg, owner, force = True)

10
roles/distgit/tasks/main.yml
View file

@ -170,15 +170,6 @@
tags:
- distgit
- name: install the pkgdb_sync_git_branches.py and pkgdb2-clone scripts
template: src={{item}} dest=/usr/local/bin/{{item}} owner=root group=root mode=0755
with_items:
- pkgdb_sync_git_branches.py
- pkgdb2-clone
tags:
- config
- distgit
- name: schedule the update hook check
cron: >
name="check-update-hooks" cron_file="ansible-check-update-hooks"
@ -287,7 +278,6 @@
file: path={{ item }} state=absent
with_items:
- /usr/local/bin/genacls.sh
- /usr/local/bin/genacls.pkgdb
- /etc/cron.d/genacls.cron
tags:
- config

160
roles/distgit/templates/genacls.pkgdb
View file

@ -1,160 +0,0 @@
#!/usr/bin/python -t
#
# Create an /etc/gitolog/conf/getolog.conf file with acls for dist-git
#
# Takes no arguments!
#
import copy
import grp
import sys
import requests
{% if env == 'staging' %}
VCS_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/vcs?format=json'
GRP_URL = 'https://admin.stg.fedoraproject.org/pkgdb/api/groups?format=json'
{% else %}
VCS_URL = 'https://admin.fedoraproject.org/pkgdb/api/vcs?format=json'
GRP_URL = 'https://admin.fedoraproject.org/pkgdb/api/groups?format=json'
{% endif %}
if __name__ == '__main__':
# Get the users in various groups
TRUSTED = grp.getgrnam('cvsadmin')[3]
ARM = grp.getgrnam('fedora-arm')[3]
SPARC = grp.getgrnam('fedora-sparc')[3]
IA64 = grp.getgrnam('fedora-ia64')[3]
S390 = grp.getgrnam('fedora-s390')[3]
PPC = grp.getgrnam('fedora-ppc')[3]
PROVEN = grp.getgrnam('provenpackager')[3]
# Set the active branches to create ACLs for
# Give them the git branch eqiv until pkgdb follows suite
ACTIVE = {
'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4',
'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6',
'epel7': 'epel7',
'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15':
'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19',
'f20': 'f20', 'f21': 'f21', 'f22': 'f22', 'f23': 'f23', 'f24': 'f24',
'f25': 'f25', 'f26': 'f26',
'devel': 'master', 'master': 'master'}
# Create a "regex"ish list 0f the reserved branches
RESERVED = [
'f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]',
'olpc[0-9]']
# Read the ACL information from the packageDB
data = requests.get(VCS_URL).json()
# print out our user groups
print '@admins = %s' % ' '.join(TRUSTED)
print '@provenpackager = %s' % ' '.join(PROVEN)
print '@fedora-arm = %s' % ' '.join(ARM)
print '@fedora-s390 = %s' % ' '.join(S390)
print '@fedora-ppc = %s' % ' '.join(PPC)
# Get a list of all the groups
groups = requests.get(GRP_URL).json()
for group in groups['groups']:
print '@%s = %s' % (group, ' '.join(grp.getgrnam(group)[3]))
# Give a little space before moving onto the permissions
print ''
# print our default permissions
print 'repo @all'
print ' - VREF/update-block-push-origin = @all'
print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc'
print ' R = @all'
#print ' RW private- = @all'
# dont' enable the above until we prevent building for real from private-
# XXX - Insert artificial namespaces into the set of namespaces returned
# by pkgdb. We want to create a mirror of rpms/PKG in test-rpms/PKG
# This hack occurs in two places. Here, and in the branch-creation script.
# https://github.com/fedora-infra/pkgdb2/issues/329#issuecomment-207050233
# And then, this got renamed from rpms-checks to test-rpms
# https://pagure.io/fedora-infrastructure/issue/5570
if 'rpms' in data:
data['test-rpms'] = copy.copy(data['rpms'])
# Also, modules are a thing
# https://pagure.io/fedora-infrastructure/issue/5571
if 'modules' in data:
data['test-modules'] = copy.copy(data['modules'])
if 'docker' in data:
data['test-docker'] = copy.copy(data['docker'])
# Get a list of all the packages
for key in data:
if key == 'title':
continue
acls = data[key]
pkglist = data[key].keys()
pkglist.sort()
if key != 'packageAcls':
key = '%s/' % key
else:
key = ''
for pkg in pkglist:
branchAcls = {} # Check whether we need to set separate per branch acls
buffer = [] # Buffer the output per package
masters = [] # Folks that have commit to master
writers = [] # Anybody that has write access
# Examine each branch in the package
branches = acls[pkg].keys()
branches.sort()
for branch in branches:
if not branch in ACTIVE.keys():
continue
if 'packager' in acls[pkg][branch]['commit']['groups']:
# If the packager group is defined, everyone has access
buffer.append(' RWC %s = @all' % (ACTIVE[branch]))
branchAcls.setdefault('@all', []).append(
(pkg, ACTIVE[branch])
)
if branch == 'master':
masters.append('@all')
if '@all' not in writers:
writers.append('@all')
else:
# Extract the owners
committers = []
owners = acls[pkg][branch]['commit']['people']
owners.sort()
for owner in owners:
committers.append(owner)
for group in acls[pkg][branch]['commit']['groups']:
committers.append('@%s' % group)
if branch == 'master':
masters.extend(committers)
# add all the committers to the top writers list
for committer in committers:
if not committer in writers:
writers.append(committer)
# Print the committers to the acl for this package-branch
committers = ' '.join(committers)
buffer.append(
' RWC %s = %s' % (ACTIVE[branch], committers))
branchAcls.setdefault(committers, []).append(
(pkg, ACTIVE[branch])
)
print ''
print 'repo %s%s' % (key, pkg)
print '\n'.join(buffer)
for reserved in RESERVED:
print ' - %s = @all' % reserved
print ' RWC refs/tags/ = %s' % ' '.join(writers)
if masters:
print ' RWC = %s' % ' '.join(masters)
sys.exit(0)

33
roles/distgit/templates/genacls.sh
View file

@ -1,33 +0,0 @@
#!/bin/bash -e
export GL_BINDIR=/usr/bin
TEMPDIR=`mktemp -d -p /var/tmp genacls.XXXXX`
cd $TEMPDIR
# If this fails, then the -e option will cause the whole script to quit.
/usr/local/bin/genacls.pkgdb > gitolite.conf
# Then create the repos and branches on disk (if we need any new ones)
python /usr/local/bin/pkgdb_sync_git_branches.py
# Leverage gitolite's Alias.pm feature to build backwards compat links
cat /etc/gitolite/RepoAliases.header > RepoAliases.pm
# Get all repos. Strip off 'rpms/'. Convert to perl mapping. Tack it on.
grep rpms/ ./gitolite.conf | \
sed 's/repo rpms\///g' | \
sed "s/.*/'&' => 'rpms\/&',/g" \
>> RepoAliases.pm
echo "};}1;" >> RepoAliases.pm
# With that done, move the files into place and run compile
mv gitolite.conf /etc/gitolite/conf/
chown gen-acls:gen-acls -R /etc/gitolite/conf/
mv RepoAliases.pm /etc/gitolite/RepoAliases.pm
chown gen-acls:gen-acls -R /etc/gitolite/RepoAliases.pm
HOME=/srv/git /usr/bin/gitolite compile
cd /
rm -rf $TEMPDIR
chown root:packager /etc/gitolite/conf/gitolite.conf-compiled.pm
chmod g+r /etc/gitolite/conf/gitolite.conf-compiled.pm

167
roles/distgit/templates/pkgdb2-clone
View file

@ -1,167 +0,0 @@
#!/usr/bin/env python
import re
import requests
import sys
import getpass
import pkgdb2client
import subprocess
#PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=User:Codeblock/RequestsSANDBOX'
PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=EPEL/epel7/Requests'
NEW_EPEL_VERSION = '7'
NEW_EPEL_SOURCE_BRANCH = 'f19'
RHEL_PKGS_PATH = '/var/lib/rhel/rhel' + NEW_EPEL_VERSION
MKBRANCH = '/usr/share/dist-git/mkbranch'
# parse_page :: String -> IO (Map String String)
# This returns a dictionary of {"pkg_name": "branch"}
def parse_page(url):
r = requests.get(url).json()
text = r['query']['pages'][r['query']['pages'].keys()[0]]['revisions'][0]['*']
lines = text.split("\n")
pkgs = filter(lambda y: y.startswith('| '), lines)
__pkgs_list__ = map(lambda y: ''.join(y.split())[1:].split('||'), pkgs)
pkgs_list = filter(lambda y: y[0] != 'foo', __pkgs_list__)
pkgs_dict = dict(pkgs_list)
return pkgs_dict
# is_in_rhel :: String -> IO Bool
def is_in_rhel(pkg):
with open(RHEL_PKGS_PATH) as f:
pkgs = map(lambda x: x.strip(), f.readlines())
return (pkg in pkgs)
# These tuples will be used to substitute one pattern for another.
# Every transform will be run on every branch name so be sure the
# pattern cannot match if you don't want it to be triggered.
transforms = (
(re.compile(r'^devel$'), 'master'),
(re.compile(r'-'), ''),
(re.compile(r'^fc([0-9])'), r'f\1'),
(re.compile(r'^epel([456])$'), r'el\1'),
(re.compile(r'^el([789]|[1-9][0-9])'), r'epel\1'),
)
branch_replacements = {'devel': (re.compile(r'^devel$'), 'master'),}
# generate_collection_cache :: PkgDB -> IO [String]
def generate_collection_cache(pkgdb):
raw_collections = pkgdb.get_collections(clt_status=(
'Active',
'Under Development'))
collection_cache = frozenset(map(lambda y: y['branchname'],
raw_collections['collections']))
return collection_cache
# normalize_branch :: [String] -> String -> IO (Option String)
def normalize_branch(collection_cache, branch):
# I originally had this implemented as a foldRight (which it really is).
# But Python doesn't eliminate tail calls. It probably would have been fine
# because "transforms" above is only 5 elements, but instead I will deal
# with the local mutation and wish that I had a type system to reason with.
# -rbe
norm_branch = branch.lower()
for transform in transforms:
norm_branch = re.sub(transform[0], transform[1], norm_branch)
# Ugh, here we break purity. Where is the option type when you need it?
if not (norm_branch in collection_cache):
print('Unknown collection specified: {0}'.format(branch))
return None
return norm_branch
# process_package :: PkgDB -> String -> String -> IO Bool
def process_package(pkgdb, pkg, src, dest):
print "*** Processing package: " + pkg
data = pkgdb.get_package(pkg)
pkg_list = data['packages']
maybe_source = [branch for branch in pkg_list if branch['collection']['branchname'] == src]
maybe_dest = [branch for branch in pkg_list if branch['collection']['branchname'] == dest]
if len(maybe_source) == 0:
print "Source branch `" + src + "' not found. Please "\
"branch " + pkg + " manually."
return False
if len(maybe_dest) != 0:
print "Package `" + pkg + "' was already branched for `" + dest + "'."\
" Not overwriting branch."
return False
if 'acls' not in maybe_source[0].keys():
print "No 'acls' key given to us by pkgdb. Cloning ACLs from a "\
"branch that has no ACLs due to retirement/orphan?"
return False
acls = [
acl
for acl in maybe_source[0]['acls']
if acl['fas_name'] != 'group::provenpackager'
]
for acl in acls:
pkgdb.update_acl(pkg, dest, acl['acl'], acl['status'], acl['fas_name'])
pkgdb.update_package_poc(pkg, dest, maybe_source[0]['point_of_contact'])
return True
# main :: [String] -> IO Unit
def main(args):
new_epel_requests = "epel" + NEW_EPEL_VERSION + "-requests"
if len(args) < 1 or (len(args) < 3 and args[0] != new_epel_requests) or\
len(args) > 3 or (len(args) > 1 and args[0] == new_epel_requests):
print "Usage: pkgdb2-clone " + new_epel_requests
print " - OR -"
print " pkgdb2-clone <source branch> <dest branch> <pkgs ...>"
sys.exit(1)
pkgdb = pkgdb2client.PkgDB()
username = raw_input('Username: ')
password = getpass.getpass()
pkgdb.login(username, password, True)
collection_cache = generate_collection_cache(pkgdb)
if args[0] == new_epel_requests:
pkgs = parse_page(PAGE_URL)
for key in pkgs:
if is_in_rhel(key):
continue
src_branchname = normalize_branch(collection_cache, pkgs[key])
dest_branchname = normalize_branch(collection_cache,
'epel' + NEW_EPEL_VERSION)
if not src_branchname or not dest_branchname:
print "[" + key + "] Invalid source or destination branch "\
"name, " + src_branchname + " -> " + dest_branchname
else:
if process_package(pkgdb, key, src_branchname, dest_branchname):
subprocess.call([MKBRANCH,
"-s",
NEW_EPEL_SOURCE_BRANCH,
"epel" + NEW_EPEL_VERSION,
key])
print "[" + key + "] \033[1m\033[32mSUCCESS\033[0m"
else:
print "[" + key + "] \033[1m\033[31mERROR\033[0m"
print "Done."
else:
src_branchname = normalize_branch(collection_cache, args[0])
dest_branchname = normalize_branch(collection_cache, args[1])
if not src_branchname or not dest_branchname:
print "[" + key + "] Invalid source or destination branch "\
"name, " + src_branchname + " -> " + dest_branchname
for pkg in args[2:]:
if process_package(pkgdb, key, src_branchname, dest_branchname):
print "[" + key + "] \033[1m\033[32mSUCCESS\033[0m"
else:
print "[" + key + "] \033[1m\033[31mERROR\033[0m"
if __name__ == '__main__':
main(sys.argv[1:])

266
roles/distgit/templates/pkgdb_sync_git_branches.py
View file

@ -1,266 +0,0 @@
#!/usr/bin/python -tt
# -*- coding: utf-8 -*-
"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
This script is able to query pkgdb and retrieve for all packages which active
branches should be there, browse all the git repos and find out which active
branches are missing.
It even goes one step further but actually adjusting the git repo by adding
the missing branches (or even the missing repo)
Here are the different steps of this script:
1/ Check the local repo in each namespace
2/ For each git repo, verifies if the new branch already exists otherwise
create it
"""
import copy
import itertools
import multiprocessing.pool
import os
import subprocess
import time
import requests
import fedmsg
# Do some off-the-bat configuration of fedmsg.
# 1) since this is a one-off script and not a daemon, it needs to connect
# to the fedmsg-relay process running on another node (or noone will
# hear it)
# 2) its going to use the 'shell' certificate which only 'sysadmin' has
# read access to. Contrast that with the 'scm' certificate which
# everyone in the 'packager' group has access to.
config = fedmsg.config.load_config([], None)
config['active'] = True
config['endpoints']['relay_inbound'] = config['relay_inbound']
fedmsg.init(name='relay_inbound', cert_prefix='shell', **config)
GIT_FOLDER = '/srv/git/repositories/'
NEW_BRANCH = 'f27'
MKBRANCH = '/usr/share/dist-git/mkbranch_branching'
SETUP_PACKAGE = '/usr/share/dist-git/setup_git_package'
THREADS = 20
VERBOSE = False
TEST_ONLY = False
class InternalError(Exception):
pass
class ProcessError(InternalError):
pass
def _invoke(program, args, cwd=None):
'''Run a command and raise an exception if an error occurred.
:arg program: The program to invoke
:args: List of arguments to pass to the program
raises ProcessError if there's a problem.
'''
cmdLine = [program]
cmdLine.extend(args)
if VERBOSE:
print ' '.join(cmdLine)
print ' in', cwd
program = subprocess.Popen(
cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
stdout, stderr = program.communicate()
if program.returncode != 0:
e = ProcessError()
e.returnCode = program.returncode
e.cmd = ' '.join(cmdLine)
e.cwd = cwd
e.message = 'Error, "%s" (in %r) returned %s\n stdout: %s\n stderr: %s' % (
e.cmd, e.cwd, e.returnCode, stdout, stderr)
print e.message
raise e
return stdout.strip()
def _create_branch(ns, pkgname, branch, existing_branches):
'''Create a specific branch for a package.
:arg pkgname: Name of the package to branch
:arg branch: Name of the branch to create
:arg existing_branches: A list of the branches that already exist locally.
'''
branch = branch.replace('*', '').strip()
if branch == 'master':
print 'ERROR: Proudly refusing to create master branch. Invalid repo?'
print 'INFO: Please check %s repo' % os.path.join(ns, pkgname)
return
if branch in existing_branches:
print 'ERROR: Refusing to create a branch %s that exists' % branch
return
try:
if VERBOSE:
print 'Creating branch: %s for package: %s' % (
branch, os.path.join(ns, pkgname))
if not TEST_ONLY:
_invoke(MKBRANCH, [branch, pkgname])# os.path.join(ns, pkgname)])
fedmsg.publish(
topic='branch',
modname='git',
msg=dict(
agent='pkgdb',
name=pkgname,
branch=branch,
namespace=ns,
),
)
except ProcessError, e:
if e.returnCode == 255:
# This is a warning, not an error
return
raise
def get_git_branch(el):
""" For the specified package name, check the local git and return the
list of branches found.
"""
ns, pkg = el
git_folder = os.path.join(GIT_FOLDER, ns, '%s.git' % pkg)
if not os.path.exists(git_folder):
if VERBOSE:
print 'Could not find %s' % git_folder
return set()
branches = [
lclbranch.replace('*', '').strip()
for lclbranch in _invoke('git', ['branch'], cwd=git_folder).split('\n')
]
return set(branches)
def branch_package(ns, pkgname, requested_branches, existing_branches):
'''Create all the branches that are listed in the pkgdb for a package.
:arg ns: The namespace of the package
:arg pkgname: The package to create branches for
:arg requested_branches: The branches to creates
:arg existing_branches: A list of existing local branches
'''
if VERBOSE:
print 'Fixing package %s for branches %s' % (pkgname, requested_branches)
# Create the devel branch if necessary
new_place = os.path.join(GIT_FOLDER, ns, '%s.git' % pkgname)
exists = os.path.exists(new_place)
if not exists or 'master' not in existing_branches:
if not TEST_ONLY:
try:
_invoke(SETUP_PACKAGE, [os.path.join(ns, pkgname)])
except:
return
if ns == 'rpms':
old_place = os.path.join(GIT_FOLDER, '%s.git' % pkgname)
if not os.path.exists(old_place):
os.symlink(new_place, old_place)
# SETUP_PACKAGE creates master
if 'master' in requested_branches:
requested_branches.remove('master')
fedmsg.publish(
topic='branch',
modname='git',
msg=dict(
agent='pkgdb',
name=pkgname,
branch='master',
namespace=ns,
),
)
# Create all the required branches for the package
# Use the translated branch name until pkgdb falls inline
for branch in requested_branches:
_create_branch(ns, pkgname, branch, existing_branches)
def main():
""" For each package found via pkgdb, check the local git for its
branches and fix inconsistencies.
"""
for ns in ['rpms']:#, 'modules', 'container']:
local_pkgs = set(os.listdir(os.path.join(GIT_FOLDER, ns)))
local_pkgs = set([it.replace('.git', '') for it in local_pkgs])
if VERBOSE:
print "Found %i local packages (namespace: %s)" % (
len(local_pkgs), ns)
if VERBOSE:
print "Finding the lists of local branches for local repos."
start = time.time()
if THREADS == 1:
git_branch_lookup = map(get_git_branch,
itertools.product([ns], sorted(local_pkgs)))
else:
threadpool = multiprocessing.pool.ThreadPool(processes=THREADS)
git_branch_lookup = threadpool.map(get_git_branch,
itertools.product([ns], sorted(local_pkgs)))
# Zip that list of results up into a lookup dict.
git_branch_lookup = dict(zip(sorted(local_pkgs), git_branch_lookup))
if VERBOSE:
print "Found all local git branches in %0.2fs" % (
time.time() - start)
tofix = set()
for pkg in sorted(local_pkgs):
git_branches = git_branch_lookup[pkg]
if NEW_BRANCH not in git_branches:
print 'Add %s to : %s' % (NEW_BRANCH, pkg)
tofix.add(pkg)
branch_package(ns, pkg, [NEW_BRANCH], git_branches)
if tofix:
print 'Packages fixed (%s): %s' % (
len(tofix), ', '.join(sorted(tofix)))
else:
if VERBOSE:
print 'Didn\'t find any packages to fix.'
if __name__ == '__main__':
import sys
sys.exit(main())

10
roles/fas_server/tasks/main.yml
View file

@ -374,16 +374,6 @@
- config
- fas
- name: HOTFIX fas2.py in python-bugzilla to add a bugzilla override for ticket 4827
copy: src={{ roles_path }}/pkgdb2/files/fas2.py
dest=/usr/lib/python2.6/site-packages/fedora/client/fas2.py
mode=644 owner=root group=root
when: master_fas_node == True
tags:
- config
- fas
- hotfixfas
- name: HOTFIX make sure only fas01 cleans up sessions
copy: src={{ roles_path }}/fas_server/files/controllers.py
dest=/usr/lib/python2.6/site-packages/fas/controllers.py

5
roles/fedmsg/irc/templates/ircbot.py
View file

@ -203,7 +203,7 @@ config = dict(
),
),
# Show only pkgdb retirement msgs and compose msgs to the releng crew.
# Show only compose msgs to the releng crew.
dict(
network='chat.freenode.net',
port=6667,
@ -218,7 +218,7 @@ config = dict(
channel='fedora-releng',
filters=dict(
topic=[
'^((?!(bodhi.mashtask.complete|pungi.compose.status.change|pkgdb\.package\.update\.status|compose.branched.complete|compose.branched.start|compose.rawhide.complete|compose.rawhide.start|bodhi.updates.|pagure)).)*$',
'^((?!(bodhi.mashtask.complete|pungi.compose.status.change|compose.branched.complete|compose.branched.start|compose.rawhide.complete|compose.rawhide.start|bodhi.updates.|pagure)).)*$',
],
body=[
"^((?!(u'success': False|u'status': u'DOOMED'|u'status': u'Retired'|u'prev_status': u'Retired'|compose|bodhi\.updates\.|\/srv\/git\/releng|'name': 'releng'|'name': 'pungi-fedora')).)*$",
@ -598,7 +598,6 @@ config = dict(
"fedoratagger": "brown",
"wiki": "purple",
"logger": "orange",
"pkgdb": "teal",
"buildsys": "yellow",
"fedoraplanet": "light green",
"trac": "pink",

1
roles/fedora_owner_change/files/fedora-owner-change.cron
View file

@ -1 +0,0 @@
0 10 * * 1 root /usr/bin/python /usr/local/bin/fedora-owner-change.py

374
roles/fedora_owner_change/files/fedora-owner-change.py
View file

@ -1,374 +0,0 @@
#!/usr/bin/python -tt
#-*- coding: utf-8 -*-
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
"""
This program checks and reports the packages owner change in pkgdb using
its messages catched by datanommer and available via datagrepper.
Dependencies:
* python-requests
* python-argparse
"""
import argparse
import json
import logging
import requests
import smtplib
import sys
from email.mime.text import MIMEText
DATAGREPPER_URL = 'https://apps.fedoraproject.org/datagrepper/raw/'
DELTA = 7 * 24 * 60 * 60 # 7 days
COMMASPACE = ', '
EMAIL_TO = ['devel@lists.fedoraproject.org', 'epel-devel@lists.fedoraproject.org']
EMAIL_FROM = 'nobody@fedoraproject.org'
SMTP_SERVER = 'localhost'
# Initial simple logging stuff
logging.basicConfig()
LOG = logging.getLogger("owner-change")
def send_report(report):
""" This function sends the actual report.
:arg report: the content to send by email
"""
report = report.encode('utf-8', 'replace')
msg = MIMEText(report)
msg['Subject'] = '[POC-change] Fedora packages point of contact updates'
msg['From'] = EMAIL_FROM
msg['To'] = COMMASPACE.join(EMAIL_TO)
# Send the message via our own SMTP server, but don't include the
# envelope header.
s = smtplib.SMTP(SMTP_SERVER)
s.sendmail(EMAIL_FROM,
EMAIL_TO,
msg.as_string())
s.quit()
def retrieve_pkgdb_change():
""" Query datagrepper to retrieve the list of change in ownership
on packages of pkgdb over the DELTA period of time.
"""
messages = []
page = 1
pages = 2
while page <= pages:
LOG.debug('Retrieving page %s of %s' % (page, pages))
data = {'delta': DELTA,
'topic': [
'org.fedoraproject.prod.pkgdb.owner.update',
'org.fedoraproject.prod.pkgdb.package.retire',
'org.fedoraproject.prod.pkgdb.package.update.status',
],
'rows_per_page': 100,
'page': page,
'order': 'asc',
}
output = requests.get(DATAGREPPER_URL, params=data)
json_output = json.loads(output.text)
pages = json_output['pages']
page += 1
messages.extend(json_output['raw_messages'])
LOG.debug('Should have retrieved %s' % json_output['total'])
return messages
def setup_parser():
"""
Set the command line arguments.
"""
parser = argparse.ArgumentParser(
prog="fedora-owner-change")
parser.add_argument(
'--nomail', action='store_true',
help="Prints the report instead of sending it by email")
parser.add_argument(
'--debug', action='store_true',
help="Outputs debugging info")
return parser
def __format_dict(dic):
keys = dic.keys()
pkgs = [it[0] for it in keys]
tmp = {}
for pkg in pkgs:
lcl_keys = [key for key in keys if pkg in key]
for key in lcl_keys:
lcl = json.dumps(dic[key])
if lcl in tmp:
tmp[lcl].append(key)
else:
tmp[lcl] = [key]
output = {}
for key in tmp:
pkg_name = tmp[key][0][0]
branches = set([val[1] for val in tmp[key]])
data = json.loads(key)
data['pkg_name'] = pkg_name
data['branches'] = ','.join(sorted(branches, key=unicode.lower))
output[pkg_name] = data
return output
def get_category(message):
""" For a given message specify if it has been orphaned, retired,
unorphaned, unretired, given or changed.
"""
output = None
poc_key = 'point_of_contact'
if 'owner' in message['package_listing']:
poc_key = 'owner'
if 'status' in message and message['status'] == 'Retired'\
and 'prev_status' in message:
output = 'retired'
elif 'retirement' in message \
and message['retirement'] == 'unretired':
output = 'unretired'
elif 'previous_owner' in message \
and message['package_listing'][poc_key] == 'orphan':
output = 'orphaned'
elif 'previous_owner' in message \
and message['package_listing'][poc_key] != 'orphan' \
and message['previous_owner'] == 'orphan':
output = 'unorphaned'
elif 'previous_owner' in message \
and message['package_listing'][poc_key] == message['previous_owner'] \
and message['previous_owner'] != 'orphan':
output = 'new'
elif 'previous_owner' in message \
and message['package_listing'][poc_key] != message['previous_owner'] \
and message['previous_owner'] != 'orphan':
output = 'given'
else:
LOG.info('Could not parse message %s', message)
return output
def main():
""" Retrieve all the change in ownership from pkgdb via datagrepper
and report the changes either as packages have been orphaned or
packages changed owner.
"""
parser = setup_parser()
args = parser.parse_args()
global LOG
if args.debug:
LOG.setLevel(logging.DEBUG)
changes = retrieve_pkgdb_change()
LOG.debug('%s changes retrieved' % len(changes))
packages = {}
for change in changes:
pkg_name = change['msg']['package_listing']['package']['name']
if 'owner' in change['msg']['package_listing']:
owner = change['msg']['package_listing']['owner']
else:
owner = change['msg']['package_listing']['point_of_contact']
branch = change['msg']['package_listing']['collection']['branchname']
user = change['msg']['agent']
LOG.debug('"%s" changed to %s by %s on %s - topic: %s' % (
pkg_name, owner, user, branch, change['topic']))
key = (pkg_name, branch)
if key not in packages:
packages[key] = {}
packages[key]['action'] = get_category(change['msg'])
packages[key]['msg'] = change['msg']
actions = {}
for act in ['orphaned', 'unorphaned', 'retired', 'unretired', 'given',
'new']:
actions[act] = {}
for package in sorted(packages) :
action = packages[package]['action']
if package[0] not in actions[action]:
actions[action][package[0]] = {}
actions[action][package[0]][package[1]] = packages[package]
hours = int(DELTA) / 3600
report = 'Change in package status over the last %s hours\n' % hours
report += '=' * (45 + len(str(hours))) + '\n'
report += '\n%s packages were orphaned\n' % len(actions['orphaned'])
report += '-' * (len(str(len(actions['orphaned']))) + 23) + '\n'
for pkg in sorted(actions['orphaned']):
branches = [item for item in actions['orphaned'][pkg]]
agents = set([
actions['orphaned'][pkg][item]['msg']['agent']
for item in actions['orphaned'][pkg]])
value = u'%s [%s] was orphaned by %s' % (
pkg,
', '.join(branches),
', '.join(agents)
)
report += value + '\n'
report += ' ' * 5 + actions['orphaned'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n%s packages were retired\n' % len(actions['retired'])
report += '-' * (len(str(len(actions['retired']))) + 23) + '\n'
for pkg in sorted(actions['retired']):
branches = [item for item in actions['retired'][pkg]]
agents = set([
actions['retired'][pkg][item]['msg']['agent']
for item in actions['retired'][pkg]])
value = u'%s [%s] was retired by %s' % (
pkg,
', '.join(branches),
', '.join(agents)
)
report += value + '\n'
report += ' ' * 5 + actions['retired'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n%s packages unorphaned\n' % len(actions['unorphaned'])
report += '-' * (len(str(len(actions['unorphaned']))) + 20) + '\n'
for pkg in sorted(actions['unorphaned']):
branches = [item for item in actions['unorphaned'][pkg]]
agents = set([
actions['unorphaned'][pkg][item]['msg']['agent']
for item in actions['unorphaned'][pkg]])
value = u'%s [%s] was unorphaned by %s' % (
pkg,
', '.join(branches),
', '.join(agents)
)
report += value + '\n'
report += ' ' * 5 + actions['unorphaned'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n%s packages were unretired\n' % len(actions['unretired'])
report += '-' * (len(str(len(actions['unretired']))) + 23) + '\n'
for pkg in sorted(actions['unretired']):
branches = [item for item in actions['unretired'][pkg]]
agents = set([
actions['unretired'][pkg][item]['msg']['agent']
for item in actions['unretired'][pkg]])
value = u'%s [%s] was unretired by %s' % (
pkg,
', '.join(branches),
', '.join(agents)
)
report += value + '\n'
report += ' ' * 5 + actions['unretired'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n%s packages were given\n' % len(actions['given'])
report += '-' * (len(str(len(actions['given']))) + 23) + '\n'
for pkg in sorted(actions['given']):
branches = [item for item in actions['given'][pkg]]
agents = set([
actions['given'][pkg][item]['msg']['agent']
for item in actions['given'][pkg]])
owners = set()
for item in actions['given'][pkg]:
if 'owner' in actions['given'][pkg][item]['msg']['package_listing']:
owners.add(
actions['given'][pkg][item]['msg'][
'package_listing']['owner']
)
else:
owners.add(
actions['given'][pkg][item]['msg'][
'package_listing']['point_of_contact']
)
value = u'%s [%s] was given by %s to %s' % (
pkg,
', '.join(branches),
', '.join(agents),
', '.join(owners)
)
report += value + '\n'
report += ' ' * 5 + actions['given'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n%s packages had new branches\n' % len(actions['new'])
report += '-' * (len(str(len(actions['new']))) + 23) + '\n'
for pkg in sorted(actions['new']):
branches = [item for item in actions['new'][pkg]]
agents = set([
actions['new'][pkg][item]['msg']['agent']
for item in actions['new'][pkg]])
owners = set()
for item in actions['new'][pkg]:
if 'owner' in actions['new'][pkg][item]['msg']['package_listing']:
owners.add(
actions['new'][pkg][item]['msg'][
'package_listing']['owner']
)
else:
owners.add(
actions['new'][pkg][item]['msg'][
'package_listing']['point_of_contact']
)
if len(branches) == 1:
branch = ': %s' % branches[0]
else:
branch = 'es: %s' % ', '.join(branches)
value = u'%s had a new branch%s for %s by %s' % (
pkg,
branch,
', '.join(owners),
', '.join(agents),
)
report += value + '\n'
report += ' ' * 5 + actions['new'][pkg][branches[0]]['msg'][
'package_listing']['package']['summary'] + '\n'
report += ' ' * 5 + 'https://admin.fedoraproject.org/pkgdb/'\
'package/%s\n' % pkg
report += '\n\nSources: https://github.com/pypingou/fedora-owner-change'
if args.nomail:
print report
else:
send_report(report)
if __name__ == '__main__':
import sys
sys.exit(main())

28
roles/fedora_owner_change/tasks/main.yml
View file

@ -1,28 +0,0 @@
---
# Cron job for the fedora-owner-change
## Install packages
- name: Install needed packages
package: name={{ item }} state=present
with_items:
- python-requests
tags:
- packages
## Install all files
- name: Install the fedora-owner-change script
copy: >
src=fedora-owner-change.py dest=/usr/local/bin/fedora-owner-change.py
owner=root group=root mode=0755
tags:
- files
## Install the cron job
- name: Install the fedora-owner-change cronjob
copy: >
src=fedora-owner-change.cron dest=/etc/cron.d/fedora-owner-change.cron
owner=root group=root mode=0644
tags:
- files

14
roles/haproxy/templates/haproxy.cfg
View file

@ -61,18 +61,6 @@ backend mirror-lists-backend
option httpchk GET /mirrorlist
option allbackups
frontend pkgdb-frontend
bind 0.0.0.0:10003
default_backend pkgdb-backend
backend pkgdb-backend
balance hdr(appserver)
server pkgdb01 pkgdb01:80 check inter 10s rise 2 fall 3
{% if env == "production" %}
server pkgdb02 pkgdb02:80 check inter 10s rise 2 fall 3
{% endif %}
option httpchk GET /pkgdb/collections/
frontend fas-frontend
bind 0.0.0.0:10004
default_backend fas-backend
@ -80,7 +68,7 @@ frontend fas-frontend
backend fas-backend
# These values are set extremely low so any issues are recovered from very
# quickly. Setting these higher will cause odd behavior in apps that
# depend on fas (like pkgdb, bodhi, etc)
# depend on fas (like bodhi, etc)
balance hdr(appserver)
server fas01 fas01:80 check inter 5s rise 1 fall 2
{% if env == "production" %}

2
roles/hosts/files/arm01-builder22.arm.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/arm01-builder23.arm.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org pkgs.stg.fedoraproject.org pkgs01.stg.phx2.fedoraproject.org

2
roles/hosts/files/buildvm-01.stg.phx2.fedoraproject.org-hosts
View file

@ -7,5 +7,5 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/buildvm-02.stg.phx2.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/buildvm-03.stg.phx2.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/buildvm-04.stg.phx2.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/buildvm-05.stg.phx2.fedoraproject.org-hosts
View file

@ -7,6 +7,6 @@
10.5.128.177 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 proxy05 proxy06 proxy07 proxy08 proxy09 proxy10 proxy11 proxy12 proxy13 proxy14 fedoraproject.org admin.fedoraproject.org admin.stg.fedoraproject.org
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.175 pkgs.fedoraproject.org

2
roles/hosts/files/composer.stg.phx2.fedoraproject.org-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-blockerbugs db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-blockerbugs db-kerneltest
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/darkserver-backend01.stg.phx2.fedoraproject.org-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/db-koji01.stg.phx2.fedoraproject.org-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest db-pps
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest db-pps
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/db-koji02.stg.phx2.fedoraproject.org-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest db-pps
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest db-pps
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/fedimg01.stg.phx2.fedoraproject.org-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.125.44 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/hotness01.stg.phx2.fedoraproject.org-hosts
View file

@ -5,6 +5,6 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.125.44 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.129 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
209.132.183.72 bugzilla.redhat.com partner-bugzilla.redhat.com

2
roles/hosts/files/koji01.stg.phx2.fedoraproject.org-hosts
View file

@ -1,4 +1,4 @@
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir db-notifs nuancier_db db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir db-notifs nuancier_db db-kerneltest

7
roles/hosts/files/pkgdb01.phx2.fedoraproject.org-hosts
View file

@ -1,7 +0,0 @@
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.5.126.23 infrastructure.fedoraproject.org
10.5.125.44 pkgs.fedoraproject.org
10.5.126.71 db-pkgdb2 db-pkgdb2

7
roles/hosts/files/pkgdb02.phx2.fedoraproject.org-hosts
View file

@ -1,7 +0,0 @@
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
10.5.126.23 infrastructure.fedoraproject.org
10.5.125.44 pkgs.fedoraproject.org
10.5.126.71 db-pkgdb2 db-pkgdb2

2
roles/hosts/files/staging-hosts
View file

@ -5,5 +5,5 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-koji01 db-github2fedmsg tagger_db db-pkgdb2 db-summershum nuancier_db db-notifs db-kerneltest db-pps
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-koji01 db-github2fedmsg tagger_dbdb-summershum nuancier_db db-notifs db-kerneltest db-pps
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all

2
roles/hosts/files/statscache-backend01.stg.phx2.fedoraproject.org-hosts
View file

@ -6,6 +6,6 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.139 koji01.stg.phx2.fedoraproject.org koji.stg.fedoraproject.org koji01 kojipkgs kojipkgs.stg.phx2.fedoraproject.org kojipkgs.stg.fedoraproject.org

2
roles/hosts/files/sundries01.stg.phx2.fedoraproject.org-hosts
View file

@ -5,7 +5,7 @@
10.5.126.23 infrastructure.fedoraproject.org
10.5.128.175 pkgs.fedoraproject.org
10.5.128.148 memcached01.stg.phx2.fedoraproject.org memcached01 memcached02 memcached03 memcached04
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_db db-pkgdb2 db-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.120 db01.stg.phx2.fedoraproject.org db-ask db-elections db-koji01 db-datanommer db-datanommer01 db-datanommer02 db-datanommer02.phx2.fedoraproject.org db-github2fedmsg db-fedocal tagger_dbdb-summershum db-tahrir nuancier_db db-notifs db-blockerbugs db-kerneltest
10.5.128.129 fas01.stg.phx2.fedoraproject.org fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all
10.5.128.139 koji01.stg.phx2.fedoraproject.org koji.stg.fedoraproject.org koji01 kojipkgs.stg.phx2.fedoraproject.org kojipkgs.stg.fedoraproject.org

2
roles/hotness/templates/hotness.py
View file

@ -76,12 +76,10 @@ config = {
'hotness.repo_url': 'https://pagure.io/releng/fedora-scm-requests',
{% if env == 'staging' %}
'hotness.pkgdb_url': 'https://admin.stg.fedoraproject.org/pkgdb/api',
"hotness.mdapi_url": "https://apps.stg.fedoraproject.org/mdapi",
'hotness.pdc_url': 'https://pdc.stg.fedoraproject.org',
'hotness.dist_git_url': 'https://src.stg.fedoraproject.org',
{% else %}
'hotness.pkgdb_url': 'https://admin.fedoraproject.org/pkgdb/api',
"hotness.mdapi_url": "https://apps.fedoraproject.org/mdapi",
'hotness.pdc_url': 'https://pdc.fedoraproject.org',
'hotness.dist_git_url': 'https://src.fedoraproject.org',

39
roles/httpd/reverseproxy/templates/reversepassproxy.pkgdb.conf
View file

@ -1,39 +0,0 @@
# Temporary fix for pkgdb performance issue. Fix for pkgdb #206
RewriteEngine On
RewriteCond %{QUERY_STRING} tg_paginate_limit=0
RewriteRule ^/pkgdb/builds/name/\* /pkgdb/builds/name/*?tg_paginate_limit=50 [R]
RequestHeader set X-Scheme https early
<Location {{localpath}}>
RequestHeader set CP-Location {{localpath}}
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css application/x-javascript
</Location>
<Location /pkgdb/static>
ExpiresActive On
ExpiresDefault "access plus 30 minutes"
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css application/x-javascript
</Location>
<Location /pkgdb/tg_js>
ExpiresActive On
ExpiresDefault "access plus 30 minutes"
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css application/x-javascript
</Location>
<Location /pkgdb/appicon>
ExpiresActive On
ExpiresDefault "access plus 30 minutes"
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css application/x-javascript
</Location>
<Location ~ {{localpath}}/(static|tg_js)>
Header unset Set-Cookie
</Location>
ProxyPass {{localpath}} {{proxyurl}}{{remotepath}}
ProxyPassReverse {{localpath}} {{proxyurl}}{{remotepath}}

1
roles/httpd/website/files/robots/robots.txt.admin.fedoraproject.org
View file

@ -1,7 +1,6 @@
User-agent: *
Disallow: /voting
Disallow: /mirrormanager
Disallow: /pkgdb
Disallow: /updates
# Temp block http://www.80legs.com/webcrawler.html

2
roles/ipsilon/templates/configuration.conf
View file

@ -57,7 +57,7 @@ openid identity url template=http://%(username)s.id{{env_suffix}}.fedoraproject.
{% if env == 'staging' %}
openid trusted roots=
{% else %}
openid trusted roots=http://jenkins.fedorainfracloud.org/securityRealm/finishLogin,http://jenkins.cloud.fedoraproject.org/securityRealm/finishLogin,https://ask.fedoraproject.org/,https://fedorahosted.org/,https://badges.fedoraproject.org,https://apps.fedoraproject.org/tagger/,https://apps.fedoraproject.org/nuancier/,https://apps.fedoraproject.org/datagrepper/,https://apps.fedoraproject.org/calendar/,http://apps.fedoraproject.org/notifications/,http://copr.fedoraproject.org/,https://copr.fedoraproject.org/,https://admin.fedoraproject.org/pkgdb/,https://admin.fedoraproject.org/voting/,https://apps.fedoraproject.org/github2fedmsg,https://admin.fedoraproject.org,https://apps.fedoraproject.org/,https://release-monitoring.org/,http://pagure.io/,http://admin.fedoraproject.org/mirrormanager/,https://apps.fedoraproject.org/koschei/,https://bodhi.fedoraproject.org,https://lists.fedoraproject.org/,https://openqa.fedoraproject.org/,https://src.fedoraproject.org/
openid trusted roots=http://jenkins.fedorainfracloud.org/securityRealm/finishLogin,http://jenkins.cloud.fedoraproject.org/securityRealm/finishLogin,https://ask.fedoraproject.org/,https://fedorahosted.org/,https://badges.fedoraproject.org,https://apps.fedoraproject.org/tagger/,https://apps.fedoraproject.org/nuancier/,https://apps.fedoraproject.org/datagrepper/,https://apps.fedoraproject.org/calendar/,http://apps.fedoraproject.org/notifications/,http://copr.fedoraproject.org/,https://copr.fedoraproject.org/,https://admin.fedoraproject.org/voting/,https://apps.fedoraproject.org/github2fedmsg,https://admin.fedoraproject.org,https://apps.fedoraproject.org/,https://release-monitoring.org/,http://pagure.io/,http://admin.fedoraproject.org/mirrormanager/,https://apps.fedoraproject.org/koschei/,https://bodhi.fedoraproject.org,https://lists.fedoraproject.org/,https://openqa.fedoraproject.org/,https://src.fedoraproject.org/
{% endif %}
openid database url=postgresql://{{ ipsilon_db_user }}:{{ ipsilon_db_pass }}@{{ ipsilon_db_host }}/{{ ipsilon_db_openid_name }}
openid untrusted roots=

2
roles/mdapi/templates/mdapi.cfg
View file

@ -4,11 +4,9 @@ PREFIX = '/mdapi'
{% if env == 'staging' %}
KOJI_REPO = 'https://koji.stg.fedoraproject.org/repos/'
PKGDB2_URL = 'https://admin.stg.fedoraproject.org/pkgdb/'
DL_SERVER = 'http://dl.phx2.fedoraproject.org'
{% else %}
KOJI_REPO = 'https://koji.fedoraproject.org/repos/'
PKGDB2_URL = 'https://admin.fedoraproject.org/pkgdb/'
DL_SERVER = 'http://dl.phx2.fedoraproject.org'
{% endif %}

2
roles/notifs/frontend/templates/fmn.web.py
View file

@ -33,10 +33,8 @@ config = {
"fmn.rules.utils.use_pagure_for_ownership": True,
{% if env == 'staging' %}
"fmn.rules.utils.pagure_api_url": "https://src.stg.fedoraproject.org/api/",
"fmn.rules.utils.pkgdb_url": "https://admin.stg.fedoraproject.org/pkgdb/api",
{% else %}
'fmn.rules.utils.pagure_api_url': 'https://src.fedoraproject.org/api/',
"fmn.rules.utils.pkgdb_url": "https://pkgdb01.phx2.fedoraproject.org/pkgdb/api",
{% endif %}
"fmn.rules.cache": {

7
roles/pdc/backend/templates/pdcupdater.py
View file

@ -35,13 +35,6 @@ config = {
'password': '{{ fedoraDummyUserPassword }}',
},
# PkgDB details
{% if env == 'staging' %}
'pdcupdater.pkgdb_url': 'https://admin.stg.fedoraproject.org/pkgdb',
{% else %}
'pdcupdater.pkgdb_url': 'https://admin.fedoraproject.org/pkgdb',
{% endif %}
# MBS details
{% if env == 'staging' %}
'pdcupdater.mbs_url': 'https://mbs.stg.fedoraproject.org/module-build-service/2/module-builds/',

951
roles/pkgdb2/files/fas2.py
View file

@ -1,951 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008-2012 Ricky Zhou, Red Hat, Inc.
# This file is part of python-fedora
#
# python-fedora is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# python-fedora is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with python-fedora; if not, see <http://www.gnu.org/licenses/>
#
'''
Provide a client module for talking to the Fedora Account System.
.. moduleauthor:: Ricky Zhou <ricky@fedoraproject.org>
.. moduleauthor:: Toshio Kuratomi <tkuratom@redhat.com>
.. moduleauthor:: Ralph Bean <rbean@redhat.com>
'''
import itertools
import urllib
import warnings
from bunch import Bunch
from kitchen.text.converters import to_bytes
try:
import libravatar
except ImportError:
libravatar = None
try:
from hashlib import md5
except ImportError:
from md5 import new as md5
from fedora.client import (
AppError, BaseClient, FasProxyClient,
FedoraClientError, FedoraServiceError
)
from fedora import __version__
### FIXME: To merge:
# /usr/bin/fasClient from fas
# API from Will Woods
# API from MyFedora
class FASError(FedoraClientError):
'''FAS Error'''
pass
class CLAError(FASError):
'''CLA Error'''
pass
USERFIELDS = [
'affiliation', 'bugzilla_email', 'certificate_serial',
'comments', 'country_code', 'creation', 'email', 'emailtoken',
'facsimile', 'gpg_keyid', 'human_name', 'id', 'internal_comments',
'ircnick', 'latitude', 'last_seen', 'longitude', 'password',
'password_changed', 'passwordtoken', 'postal_address', 'privacy',
'locale', 'ssh_key', 'status', 'status_change', 'telephone',
'unverified_email', 'timezone', 'username', 'security_question',
'security_answer', ]
class AccountSystem(BaseClient):
'''An object for querying the Fedora Account System.
The Account System object provides a python API for talking to the Fedora
Account System. It abstracts the http requests, cookie handling, and
other details so you can concentrate on the methods that are important to
your program.
.. warning::
If your code is trying to use the AccountSystem object to
connect to fas for multiple users you probably want to use
:class:`~fedora.client.FasProxyClient` instead. If your code is
trying to reuse a single instance of AccountSystem for multiple users
you *definitely* want to use :class:`~fedora.client.FasProxyClient`
instead. Using AccountSystem in these cases may result in a user
being logged in as a different user. (This may be the case even if
you instantiate a new AccountSystem object for each user if
:attr:cache_session: is True since that creates a file on the file
system that can end up loading session credentials for the wrong
person.
.. versionchanged:: 0.3.26
Added :meth:`~fedora.client.AccountSystem.gravatar_url` that returns
a url to a gravatar for a user.
.. versionchanged:: 0.3.33
Renamed :meth:`~fedora.client.AccountSystem.gravatar_url` to
:meth:`~fedora.client.AccountSystem.avatar_url`.
'''
# proxy is a thread-safe connection to the fas server for verifying
# passwords of other users
proxy = None
# size that we allow to request from remote avatar providers.
_valid_avatar_sizes = (32, 64, 140)
# URLs for remote avatar providers.
_valid_avatar_services = ['libravatar', 'gravatar']
def __init__(self, base_url='https://admin.fedoraproject.org/accounts/',
*args, **kwargs):
'''Create the AccountSystem client object.
:kwargs base_url: Base of every URL used to contact the server.
Defaults to the Fedora Project FAS instance.
:kwargs useragent: useragent string to use. If not given, default to
"Fedora Account System Client/VERSION"
:kwargs debug: If True, log debug information
:kwargs username: username for establishing authenticated connections
:kwargs password: password to use with authenticated connections
:kwargs session_cookie: **Deprecated** Use session_id instead.
User's session_cookie to connect to the server
:kwargs session_id: user's session_id to connect to the server
:kwargs cache_session: if set to true, cache the user's session cookie
on the filesystem between runs.
'''
if 'useragent' not in kwargs:
kwargs['useragent'] = \
'Fedora Account System Client/%s' % __version__
super(AccountSystem, self).__init__(base_url, *args, **kwargs)
# We need a single proxy for the class to verify username/passwords
# against.
if not self.proxy:
self.proxy = FasProxyClient(base_url, useragent=self.useragent,
session_as_cookie=False,
debug=self.debug,
insecure=self.insecure)
# Preseed a list of FAS accounts with bugzilla addresses
# This allows us to specify a different email for bugzilla than is
# in the FAS db. It is a hack, however, until FAS has a field for the
# bugzilla address.
self.__bugzilla_email = {
# Konstantin Ryabitsev: mricon@gmail.com
100029: 'icon@fedoraproject.org',
# Sean Reifschneider: jafo@tummy.com
100488: 'jafo-redhat@tummy.com',
# Karen Pease: karen-pease@uiowa.edu
100281: 'meme@daughtersoftiresias.org',
# Robert Scheck: redhat@linuxnetz.de
100093: 'redhat-bugzilla@linuxnetz.de',
# Scott Bakers: bakers@web-ster.com
100881: 'scott@perturb.org',
# Colin Charles: byte@aeon.com.my
100014: 'byte@fedoraproject.org',
# W. Michael Petullo: mike@flyn.org
100136: 'redhat@flyn.org',
# Elliot Lee: sopwith+fedora@gmail.com
100060: 'sopwith@redhat.com',
# Control Center Team: Bugzilla user but email doesn't exist
9908: 'control-center-maint@redhat.com',
# Máirín Duffy
100548: 'duffy@redhat.com',
# Muray McAllister: murray.mcallister@gmail.com
102321: 'mmcallis@redhat.com',
# William Jon McCann: mccann@jhu.edu
102489: 'jmccann@redhat.com',
# Matt Domsch's rebuild script -- bz email goes to /dev/null
103590: 'ftbfs@fedoraproject.org',
# Sindre Pedersen Bjørdal: foolish@guezz.net
100460: 'sindrepb@fedoraproject.org',
# Jesus M. Rodriguez: jmrodri@gmail.com
102180: 'jesusr@redhat.com',
# Roozbeh Pournader: roozbeh@farsiweb.info
100350: 'roozbeh@gmail.com',
# Michael DeHaan: michael.dehaan@gmail.com
100603: 'mdehaan@redhat.com',
# Sebastian Gosenheimer: sgosenheimer@googlemail.com
103647: 'sebastian.gosenheimer@proio.com',
# Ben Konrath: bkonrath@redhat.com
101156: 'ben@bagu.org',
# Kai Engert: kaie@redhat.com
100399: 'kengert@redhat.com',
# William Jon McCann: william.jon.mccann@gmail.com
102952: 'jmccann@redhat.com',
# Simon Wesp: simon@w3sp.de
109464: 'cassmodiah@fedoraproject.org',
# Robert M. Albrecht: romal@gmx.de
101475: 'mail@romal.de',
# Davide Cescato: davide.cescato@iaeste.ch
123204: 'ceski@fedoraproject.org',
# Nick Bebout: nick@bebout.net
101458: 'nb@fedoraproject.org',
# Niels Haase: haase.niels@gmail.com
126862: 'arxs@fedoraproject.org',
# Thomas Janssen: th.p.janssen@googlemail.com
103110: 'thomasj@fedoraproject.org',
# Michael J Gruber: 'michaeljgruber+fedoraproject@gmail.com'
105113: 'mjg@fedoraproject.org',
# Juan Manuel Rodriguez Moreno: 'nushio@gmail.com'
101302: 'nushio@fedoraproject.org',
# Andrew Cagney: 'andrew.cagney@gmail.com'
102169: 'cagney@fedoraproject.org',
# Jeremy Katz: 'jeremy@katzbox.net'
100036: 'katzj@fedoraproject.org',
# Dominic Hopf: 'dmaphy@gmail.com'
124904: 'dmaphy@fedoraproject.org',
# Christoph Wickert: 'christoph.wickert@googlemail.com':
100271: 'cwickert@fedoraproject.org',
# Elliott Baron: 'elliottbaron@gmail.com'
106760: 'ebaron@fedoraproject.org',
# Thomas Spura: 'spurath@students.uni-mainz.de'
111433: 'tomspur@fedoraproject.org',
# Adam Miller: 'maxamillion@gmail.com'
110673: 'admiller@redhat.com',
# Garrett Holmstrom: 'garrett.holmstrom@gmail.com'
131739: 'gholms@fedoraproject.org',
# Tareq Al Jurf: taljurf.fedora@gmail.com
109863: 'taljurf@fedoraproject.org',
# Josh Kayse: jokajak@gmail.com
148243: 'jokajak@fedoraproject.org',
# Behdad Esfahbod: fedora@behdad.org
100102: 'behdad@fedoraproject.org',
# Daniel Bruno: danielbrunos@gmail.com
101608: 'dbruno@fedoraproject.org',
# Beth Lynn Eicher: bethlynneicher@gmail.com
148706: 'bethlynn@fedoraproject.org',
# Andre Robatino: andre.robatino@verizon.net
114970: 'robatino@fedoraproject.org',
# Jeff Sheltren: jeff@tag1consulting.com
100058: 'sheltren@fedoraproject.org',
# Josh Boyer: jwboyer@gmail.com
100115: 'jwboyer@redhat.com',
# Matthew Miller: mattdm@mattdm.org
100042: 'mattdm@redhat.com',
# Jamie Nguyen: j@jamielinux.com
160587: 'jamielinux@fedoraproject.org',
# Nikos Roussos: nikos@roussos.cc
144436: 'comzeradd@fedoraproject.org',
# Benedikt Schäfer: benedikt@schaefer-flieden.de
154726: 'ib54003@fedoraproject.org',
# Ricky Elrod: codeblock@elrod.me
139137: 'relrod@redhat.com',
# David Xie: david.scriptfan@gmail.com
167133: 'davidx@fedoraproject.org',
# Felix Schwarz: felix.schwarz@oss.schwarz.eu
103551: 'fschwarz@fedoraproject.org',
# Martin Holec: martix@martix.names
137561: 'mholec@redhat.com',
# John Dulaney: j_dulaney@live.com
149140: 'jdulaney@fedoraproject.org',
# Niels de Vos: niels@nixpanic.net
102792: 'ndevos@redhat.com',
# Shawn Wells: shawn@redhat.com
156515: 'swells@redhat.com',
# Christopher Tubbs: ctubbsii+fedora@gmail.com
160404: 'ctubbsii@fedoraproject.org',
# Björn Esser: bjoern.esser@gmail.com
163460: 'besser82@fedoraproject.org',
# Amit Shah: amit@amitshah.net
115389: 'amitshah@fedoraproject.org',
# Mark Wielard: fedora@wildebeest.org
102697: 'mjw@fedoraproject.org',
# Benjamin Lefoul: benjamin.lefoul@nwise.se
189661: 'lef@fedoraproject.org',
# Mike Ruckman: roshi@mykolab.com
172063: 'roshi@fedoraproject.org',
}
# A few people have an email account that is used in owners.list but
# have setup a bugzilla account for their primary account system email
# address now. Map these here.
self.__alternate_email = {
# Damien Durand: splinux25@gmail.com
'splinux@fedoraproject.org': 100406,
# Kevin Fenzi: kevin@tummy.com
'kevin-redhat-bugzilla@tummy.com': 100037,
}
for bugzilla_map in self.__bugzilla_email.items():
self.__alternate_email[bugzilla_map[1]] = bugzilla_map[0]
# We use the two mappings as follows::
# When looking up a user by email, use __alternate_email.
# When looking up a bugzilla email address use __bugzilla_email.
#
# This allows us to parse in owners.list and have a value for all the
# emails in there while not using the alternate email unless it is
# the only option.
# TODO: Use exceptions properly
### Set insecure properly ###
# When setting insecure, we have to set it both on ourselves and on
# self.proxy
def _get_insecure(self):
return self._insecure
def _set_insecure(self, insecure):
self._insecure = insecure
self.proxy = FasProxyClient(self.base_url, useragent=self.useragent,
session_as_cookie=False, debug=self.debug,
insecure=insecure)
return insecure
#: If this attribute is set to True, do not check server certificates
#: against their CA's. This means that man-in-the-middle attacks are
#: possible. You might turn this option on for testing against a local
#: version of a server with a self-signed certificate but it should be off
#: in production.
insecure = property(_get_insecure, _set_insecure)
### Groups ###
def create_group(self, name, display_name, owner, group_type,
invite_only=0, needs_sponsor=0, user_can_remove=1,
prerequisite='', joinmsg='', apply_rules='None'):
'''Creates a FAS group.
:arg name: The short group name (alphanumeric only).
:arg display_name: A longer version of the group's name.
:arg owner: The username of the FAS account which owns the new group.
:arg group_type: The kind of group being created. Current valid options
are git, svn, hg, shell, and tracking.
:kwarg invite_only: Users must be invited to the group, they cannot
join on their own.
:kwarg needs_sponsor: Users must be sponsored into the group.
:kwarg user_can_remove: Users can remove themselves from the group.
:kwarg prerequisite: Users must be in the given group (string) before
they can join the new group.
:kwarg joinmsg: A message shown to users when they apply to the group.
:kwarg apply_rules: Rules for applying to the group, shown to users
before they apply.
:rtype: :obj:`bunch.Bunch`
:returns: A Bunch containing information about the group that was
created.
.. versionadded:: 0.3.29
'''
req_params = {
'invite_only': invite_only,
'needs_sponsor': needs_sponsor,
'user_can_remove': user_can_remove,
'prerequisite': prerequisite,
'joinmsg': joinmsg,
'apply_rules': apply_rules
}
request = self.send_request(
'/group/create/%s/%s/%s/%s' % (
urllib.quote(name),
urllib.quote(display_name),
urllib.quote(owner),
urllib.quote(group_type)),
req_params=req_params,
auth=True
)
return request
def group_by_id(self, group_id):
'''Returns a group object based on its id'''
params = {'group_id': int(group_id)}
request = self.send_request(
'json/group_by_id',
auth=True,
req_params=params
)
if request['success']:
return request['group']
else:
return dict()
def group_by_name(self, groupname):
'''Returns a group object based on its name'''
params = {'groupname': groupname}
request = self.send_request(
'json/group_by_name',
auth=True,
req_params=params
)
if request['success']:
return request['group']
else:
raise AppError(
message='FAS server unable to retrieve group'
' %(group)s' % {'group': to_bytes(groupname)},
name='FASError')
def group_members(self, groupname):
'''Return a list of people approved for a group.
This method returns a list of people who are in the requested group.
The people are all approved in the group. Unapproved people are not
shown. The format of data is::
\[{'username': 'person1', 'role_type': 'user'},
\{'username': 'person2', 'role_type': 'sponsor'}]
role_type can be one of 'user', 'sponsor', or 'administrator'.
.. versionadded:: 0.3.2
.. versionchanged:: 0.3.21
Return a Bunch instead of a DictContainer
'''
request = self.send_request('/group/dump/%s' %
urllib.quote(groupname), auth=True)
return [Bunch(username=user[0],
role_type=user[3]) for user in request['people']]
### People ###
def person_by_id(self, person_id):
'''Returns a person object based on its id'''
person_id = int(person_id)
params = {'person_id': person_id}
request = self.send_request('json/person_by_id', auth=True,
req_params=params)
if request['success']:
if person_id in self.__bugzilla_email:
request['person']['bugzilla_email'] = \
self.__bugzilla_email[person_id]
else:
request['person']['bugzilla_email'] = \
request['person']['email']
# In a devel version of FAS, membership info was returned
# separately
# This was later corrected (can remove this code at some point)
if 'approved' in request:
request['person']['approved_memberships'] = request['approved']
if 'unapproved' in request:
request['person']['unapproved_memberships'] = \
request['unapproved']
return request['person']
else:
return dict()
def person_by_username(self, username):
'''Returns a person object based on its username'''
params = {'username': username}
request = self.send_request(
'json/person_by_username',
auth=True,
req_params=params)
if request['success']:
person = request['person']
if person['id'] in self.__bugzilla_email:
person['bugzilla_email'] = self.__bugzilla_email[person['id']]
else:
person['bugzilla_email'] = person['email']
# In a devel version of FAS, membership info was returned
# separately
# This was later corrected (can remove this code at some point)
if 'approved' in request:
request['person']['approved_memberships'] = request['approved']
if 'unapproved' in request:
request['person']['unapproved_memberships'] = \
request['unapproved']
return person
else:
return dict()
def avatar_url(self, username, size=64,
default=None, lookup_email=True,
service=None):
''' Returns a URL to an avatar for a given username.
Avatars are drawn from third party services.
:arg username: FAS username to construct a avatar url for
:kwarg size: size of the avatar. Allowed sizes are 32, 64, 140.
Default: 64
:kwarg default: If the service does not have a avatar image for the
email address, this url is returned instead. Default:
the fedora logo at the specified size.
:kwarg lookup_email: If true, use the email from FAS for gravatar.com
lookups, otherwise just append @fedoraproject.org to the username.
For libravatar.org lookups, this is ignored. The openid identifier
of the user is used instead.
Note that gravatar.com lookups will be much slower if lookup_email
is set to True since we'd have to make a query against FAS itself.
:kwarg service: One of 'libravatar' or 'gravatar'.
Default: 'libravatar'.
:raises ValueError: if the size parameter is not allowed or if the
service is not one of 'libravatar' or 'gravatar'
:rtype: :obj:`str`
:returns: url of a avatar for the user
If that user has no avatar entry, instruct the remote service to
redirect us to the Fedora logo.
If that user has no email attribute, then make a fake request to
the third party service.
.. versionadded:: 0.3.26
.. versionchanged: 0.3.30
Add lookup_email parameter to control whether we generate avatar
urls with the email in fas or username@fedoraproject.org
.. versionchanged: 0.3.33
Renamed from `gravatar_url` to `avatar_url`
.. versionchanged: 0.3.34
Updated libravatar to use the user's openid identifier.
'''
if size not in self._valid_avatar_sizes:
raise ValueError(
'Size %(size)i disallowed. Must be in %(valid_sizes)r' % {
'size': size,
'valid_sizes': self._valid_avatar_sizes
}
)
# If our caller explicitly requested libravatar but they don't have
# it installed, then we need to raise a nice error and let them know.
if service == 'libravatar' and not libravatar:
raise ValueError("Install python-pylibravatar if you want to "
"use libravatar as an avatar provider.")
# If our caller didn't specify a service, let's pick a one for them.
# If they have pylibravatar installed, then by all means let freedom
# ring! Otherwise, we'll use gravatar.com if we have to.
if not service:
if libravatar:
service = 'libravatar'
else:
service = 'gravatar'
# Just double check to make sure they didn't pass us a bogus service.
if service not in self._valid_avatar_services:
raise ValueError(
'Service %(service)r disallowed. '
'Must be in %(valid_services)r' % {
'service': service,
'valid_services': self._valid_avatar_services
}
)
if not default:
default = "http://fedoraproject.org/static/images/" + \
"fedora_infinity_%ix%i.png" % (size, size)
if service == 'libravatar':
openid = 'http://%s.id.fedoraproject.org/' % username
return libravatar.libravatar_url(
openid=openid,
size=size,
default=default,
)
else:
if lookup_email:
person = self.person_by_username(username)
email = person.get('email', 'no_email')
else:
email = "%s@fedoraproject.org" % username
query_string = urllib.urlencode({
's': size,
'd': default,
})
hash = md5(email).hexdigest()
return "http://www.gravatar.com/avatar/%s?%s" % (
hash, query_string)
def gravatar_url(self, *args, **kwargs):
""" *Deprecated* - Use avatar_url.
.. versionadded:: 0.3.26
.. versionchanged: 0.3.30
Add lookup_email parameter to control whether we generate gravatar
urls with the email in fas or username@fedoraproject.org
.. versionchanged: 0.3.33
Deprecated in favor of `avatar_url`.
"""
warnings.warn(
"gravatar_url is deprecated and will be removed in"
" a future version. Please port your code to use avatar_url(...,"
" service='libravatar', ...) instead",
DeprecationWarning, stacklevel=2)
if 'service' in kwargs:
raise TypeError("'service' is an invalid keyword argument for"
" this function. Use avatar_url() instead)")
return self.avatar_url(*args, service='gravatar', **kwargs)
def user_id(self):
'''Returns a dict relating user IDs to usernames'''
request = self.send_request('json/user_id', auth=True)
people = {}
for person_id, username in request['people'].items():
# change userids from string back to integer
people[int(person_id)] = username
return people
def people_by_key(self, key=u'username', search=u'*', fields=None):
'''Return a dict of people
:kwarg key: Key by this field. Valid values are 'id', 'username', or
'email'. Default is 'username'
:kwarg search: Pattern to match usernames against. Defaults to the
'*' wildcard which matches everyone.
:kwarg fields: Limit the data returned to a specific list of fields.
The default is to retrieve all fields.
Valid fields are:
* affiliation
* alias_enabled
* bugzilla_email
* certificate_serial
* comments
* country_code
* creation
* email
* emailtoken
* facsimile
* gpg_keyid
* group_roles
* human_name
* id
* internal_comments
* ircnick
* last_seen
* latitude
* locale
* longitude
* memberships
* old_password
* password
* password_changed
* passwordtoken
* postal_address
* privacy
* roles
* ssh_key
* status
* status_change
* telephone
* timezone
* unverified_email
* username
Note that for most users who access this data, many of these
fields will be set to None due to security or privacy settings.
:returns: a dict relating the key value to the fields.
.. versionchanged:: 0.3.21
Return a Bunch instead of a DictContainer
.. versionchanged:: 0.3.26
Fixed to return a list with both people who have signed the CLA
and have not
'''
# Make sure we have a valid key value
if key not in ('id', 'username', 'email'):
raise KeyError('key must be one of "id", "username", or'
' "email"')
if fields:
fields = list(fields)
for field in fields:
if field not in USERFIELDS:
raise KeyError('%(field)s is not a valid field to'
' filter' % {'field': to_bytes(field)})
else:
fields = USERFIELDS
# Make sure we retrieve the key value
unrequested_fields = []
if key not in fields:
unrequested_fields.append(key)
fields.append(key)
if 'bugzilla_email' in fields:
# Need id and email for the bugzilla information
if 'id' not in fields:
unrequested_fields.append('id')
fields.append('id')
if 'email' not in fields:
unrequested_fields.append('email')
fields.append('email')
request = self.send_request(
'/user/list',
req_params={
'search': search,
'fields': [f for f in fields if f != 'bugzilla_email']
},
auth=True)
people = Bunch()
for person in itertools.chain(request['people'],
request['unapproved_people']):
# Retrieve bugzilla_email from our list if necessary
if 'bugzilla_email' in fields:
if person['id'] in self.__bugzilla_email:
person['bugzilla_email'] = \
self.__bugzilla_email[person['id']]
else:
person['bugzilla_email'] = person['email']
person_key = person[key]
# Remove any fields that weren't requested by the user
if unrequested_fields:
for field in unrequested_fields:
del person[field]
# Add the person record to the people dict
people[person_key] = person
return people
def people_by_id(self):
'''*Deprecated* Use people_by_key() instead.
Returns a dict relating user IDs to human_name, email, username,
and bugzilla email
.. versionchanged:: 0.3.21
Return a Bunch instead of a DictContainer
'''
warnings.warn(
"people_by_id() is deprecated and will be removed in"
" 0.4. Please port your code to use people_by_key(key='id',"
" fields=['human_name', 'email', 'username', 'bugzilla_email'])"
" instead", DeprecationWarning, stacklevel=2)
request = self.send_request('/json/user_id', auth=True)
user_to_id = {}
people = Bunch()
for person_id, username in request['people'].items():
person_id = int(person_id)
# change userids from string back to integer
people[person_id] = {'username': username, 'id': person_id}
user_to_id[username] = person_id
# Retrieve further useful information about the users
request = self.send_request('/group/dump', auth=True)
for user in request['people']:
userid = user_to_id[user[0]]
person = people[userid]
person['email'] = user[1]
person['human_name'] = user[2]
if userid in self.__bugzilla_email:
person['bugzilla_email'] = self.__bugzilla_email[userid]
else:
person['bugzilla_email'] = person['email']
return people
### Utils ###
def people_by_groupname(self, groupname):
'''Return a list of persons for the given groupname.
:arg groupname: Name of the group to look up
:returns: A list of person objects from the group. If the group
contains no entries, then an empty list is returned.
'''
people = self.people_by_id()
group = dict(self.group_by_name(groupname))
userids = [user[u'person_id'] for user in
group[u'approved_roles'] + group[u'unapproved_roles']]
return [people[userid] for userid in userids]
### Configs ###
def get_config(self, username, application, attribute):
'''Return the config entry for the key values.
:arg username: Username of the person
:arg application: Application for which the config is set
:arg attribute: Attribute key to lookup
:raises AppError: if the server returns an exception
:returns: The unicode string that describes the value. If no entry
matched the username, application, and attribute then None is
returned.
'''
request = self.send_request('config/list/%s/%s/%s' %
(username, application, attribute),
auth=True)
if 'exc' in request:
raise AppError(
name=request['exc'],
message=request['tg_flash']
)
# Return the value if it exists, else None.
if 'configs' in request and attribute in request['configs']:
return request['configs'][attribute]
return None
def get_configs_like(self, username, application, pattern=u'*'):
'''Return the config entries that match the keys and the pattern.
Note: authentication on the server will prevent anyone but the user
or a fas admin from viewing or changing their configs.
:arg username: Username of the person
:arg application: Application for which the config is set
:kwarg pattern: A pattern to select values for. This accepts * as a
wildcard character. Default='*'
:raises AppError: if the server returns an exception
:returns: A dict mapping ``attribute`` to ``value``.
'''
request = self.send_request(
'config/list/%s/%s/%s' %
(username, application, pattern),
auth=True)
if 'exc' in request:
raise AppError(
name=request['exc'],
message=request['tg_flash'])
return request['configs']
def set_config(self, username, application, attribute, value):
'''Set a config entry in FAS for the user.
Note: authentication on the server will prevent anyone but the user
or a fas admin from viewing or changing their configs.
:arg username: Username of the person
:arg application: Application for which the config is set
:arg attribute: The name of the config key that we're setting
:arg value: The value to set this to
:raises AppError: if the server returns an exception
'''
request = self.send_request(
'config/set/%s/%s/%s' %
(username, application, attribute),
req_params={'value': value}, auth=True)
if 'exc' in request:
raise AppError(
name=request['exc'],
message=request['tg_flash'])
def people_query(self, constraints=None, columns=None):
'''Returns a list of dicts representing database rows
:arg constraints: A dictionary specifying WHERE constraints on columns
:arg columns: A list of columns to be selected in the query
:raises AppError: if the query failed on the server (most likely
because the server was given a bad query)
:returns: A list of dicts representing database rows (the keys of
the dict are the columns requested)
.. versionadded:: 0.3.12.1
'''
if constraints is None:
constraints = {}
if columns is None:
columns = []
req_params = {}
req_params.update(constraints)
req_params['columns'] = ','.join(columns)
try:
request = self.send_request(
'json/people_query',
req_params=req_params, auth=True)
if request['success']:
return request['data']
else:
raise AppError(message=request['error'], name='FASError')
except FedoraServiceError:
raise
### Certs ###
def user_gencert(self):
'''Generate a cert for a user'''
try:
request = self.send_request('user/dogencert', auth=True)
except FedoraServiceError:
raise
if not request['cla']:
raise CLAError
return "%(cert)s\n%(key)s" % request
### Passwords ###
def verify_password(self, username, password):
'''Return whether the username and password pair are valid.
:arg username: username to try authenticating
:arg password: password for the user
:returns: True if the username/password are valid. False otherwise.
'''
return self.proxy.verify_password(username, password)
### fasClient Special Methods ###
def group_data(self, force_refresh=None):
'''Return administrators/sponsors/users and group type for all groups
:arg force_refresh: If true, the returned data will be queried from the
database, as opposed to memcached.
:raises AppError: if the query failed on the server
:returns: A dict mapping group names to the group type and the
user IDs of the administrator, sponsors, and users of the group.
.. versionadded:: 0.3.8
'''
params = {}
if force_refresh:
params['force_refresh'] = True
try:
request = self.send_request(
'json/fas_client/group_data',
req_params=params, auth=True)
if request['success']:
return request['data']
else:
raise AppError(
message='FAS server unable to retrieve'
' group members', name='FASError')
except FedoraServiceError:
raise
def user_data(self):
'''Return user data for all users in FAS
Note: If the user is not authorized to see password hashes,
'*' is returned for the hash.
:raises AppError: if the query failed on the server
:returns: A dict mapping user IDs to a username, password hash,
SSH public key, email address, and status.
.. versionadded:: 0.3.8
'''
try:
request = self.send_request('json/fas_client/user_data', auth=True)
if request['success']:
return request['data']
else:
raise AppError(
message='FAS server unable to retrieve user'
' information', name='FASError')
except FedoraServiceError:
raise

105
roles/pkgdb2/tasks/main.yml
View file

@ -1,105 +0,0 @@
---
# Configuration for the pkgdb2 webapp
- name: install needed packages
package: name={{ item }} state=present
with_items:
- pkgdb2
- python-psycopg2
- python-openid-cla
- python-openid-teams
- python-memcached
- libsemanage-python
- pyliblzma
- python-bunch
tags:
- packages
- pkgdb
# HOTFIX: adjust bugzilla overrides
- name: HOTFIX - adjust bugzilla overrides
copy: src=fas2.py dest=/usr/lib/python2.7/site-packages/fedora/client/fas2.py
tags:
- config
- pkgdb
- fas
- hotfix
- hotfixfas
- name: copy sundry pkgdb configuration
template: src={{ item.file }}
dest={{ item.location }}/{{ item.dest }}
owner=apache group=apache mode=0600
with_items:
- { file: pkgdb2_admin.cfg, location: /etc/pkgdb2, dest: pkgdb2.cfg }
- { file: alembic.ini, location: /etc/pkgdb2, dest: alembic.ini }
changed_when: "1 != 1"
tags:
- config
- pkgdb
notify:
- restart apache
- name: create the database scheme
command: /usr/bin/python2 /usr/share/pkgdb2/pkgdb2_createdb.py
changed_when: "1 != 1"
environment:
PKGDB2_CONFIG: /etc/pkgdb2/pkgdb2.cfg
tags: pkgdb
- name: Install all the configuration file of pkgdb2
template: src={{ item.file }}
dest={{ item.location }}/{{ item.file }}
owner=apache group=apache mode=0600
with_items:
- { file: pkgdb2.cfg, location: /etc/pkgdb2 }
- { file: pkgdb2.conf, location: /etc/httpd/conf.d }
- { file: pkgdb2.wsgi, location: /var/www/, dest: pkgdb2.wsgi }
tags:
- config
- pkgdb
notify:
- restart apache
- name: Remove old cronjobs.
file:
path: "{{ item.location }}/{{ item.file }}"
state: absent
with_items:
- { file: 'pkgdb-sync-bugzilla.cron', location: /etc/cron.d }
- { file: 'pkgdb_update_package_info.cron', location: /etc/cron.d }
tags:
- config
- pkgdb
- cron
- name: update the base_url in update_package_info
command: sed -i -e 's|https://dl.fedoraproject.org|http://dl.phx2.fedoraproject.org|' /usr/bin/update_package_info.py
changed_when: "1 != 1"
tags:
- config
- pkgdb
- name: set sebooleans so pkgdb2 can talk to the db
seboolean: name=httpd_can_network_connect_db
state=true
persistent=true
tags:
- selinux
- pkgdb
- name: set sebooleans so apache can send emails
seboolean: name=httpd_can_sendmail
state=true
persistent=true
tags:
- selinux
- pkgdb
- name: set sebooleans so apache can talk to memcached0*
seboolean: name=httpd_can_network_memcache
state=true
persistent=true
tags:
- selinux
- pkgdb

51
roles/pkgdb2/templates/alembic.ini
View file

@ -1,51 +0,0 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = /usr/share/pkgdb2/alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
#sqlalchemy.url = postgresql://<%= pkgdb_app %>:<%= pkgdb_appPassword %>@db-pkgdb/pkgdb
sqlalchemy.url = postgresql://{{ pkgdb2_db_admin_user }}:{{ pkgdb2_db_admin_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

4
roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron
View file

@ -1,4 +0,0 @@
# Synchronize pkgdb with bugzilla
#
0 */2 * * * root PKGDB2_CONFIG=/etc/pkgdb2/pkgdb2.cfg /usr/local/bin/lock-wrapper pkgdb-sync-bugzilla "/usr/bin/pkgdb-sync-bugzilla"

203
roles/pkgdb2/templates/pkgdb2.cfg
View file

@ -1,203 +0,0 @@
# Beware that the quotes around the values are mandatory
### Secret key for the Flask application
SECRET_KEY='{{ pkgdb2_secret_key }}'
### url to the database server:
#DB_URL=mysql://user:pass@host/db_name
#DB_URL=postgres://user:pass@host/db_name
DB_URL='postgresql://{{ pkgdb2_db_user }}:{{ pkgdb2_db_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}'
### the number of items (packages, packagers..) to display on the search
### pages
ITEMS_PER_PAGE = 50
### List the ACL which are automatically approved (don't need reviewing)
AUTO_APPROVE = ['watchcommits', 'watchbugzilla']
### List of FAS user that can be automatically approved w/o checking if they
### are packagers
AUTOAPPROVE_PKGERS = [
'anaconda-maint', 'astronomy-sig', 'ctrl-center-team', 'design-sw',
'fonts-sig', 'gecko-maint', 'hams-sig', 'haskell-sig', 'i18n-team',
'java-sig', 'kernel-maint', 'lvm-team', 'ml-sig', 'mono-sig', 'perl-sig',
'virtmain', 'xen-maint', 'xgl-maint',
]
#### FAS group for the pkgdb admins
{% if env == 'staging' %}
# Factory 2 added to make it easier for us to create repos for the modularity
# team. Only in staging. Feel free to remove!
ADMIN_GROUP = ['cvsadmin', 'factory2']
{% else %}
ADMIN_GROUP = ['sysadmin-main', 'cvsadmin']
{% endif %}
## URLs to fedmenu resources
{% if env == 'staging' %}
FEDMENU_URL = 'https://apps.stg.fedoraproject.org/fedmenu'
FEDMENU_DATA_URL = 'https://apps.stg.fedoraproject.org/js/data.js'
{% else %}
FEDMENU_URL = 'https://apps.fedoraproject.org/fedmenu'
FEDMENU_DATA_URL = 'https://apps.fedoraproject.org/js/data.js'
{% endif %}
### The default backend for dogpile
### Options are listed at:
### http://dogpilecache.readthedocs.org/en/latest/api.html (backend section)
PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached'
PKGDB2_CACHE_KWARGS = {
'arguments': {
'url': ["memcached01:11211"],
'distributed_lock': True,
}
}
### Bugzilla information
## Upon changes in pkgdb, update bugzilla
# PKGDB2_BUGZILLA_NOTIFICATION = False
## URL to the bugzilla instance to update
PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com'
## name of the user the pkgdb application can log in to bugzilla with
PKGDB2_BUGZILLA_USER = '{{ bugzilla_user }}'
## password of the user the pkgdb application can log in to bugzilla with
PKGDB2_BUGZILLA_PASSWORD = '{{ bugzilla_password }}'
# Settings specific to the ``pkgdb-sync-bugzilla`` script/cron
PKGDB2_BUGZILLA_NOTIFY_EMAIL = [
'kevin@fedoraproject.org',
'pingou@fedoraproject.org']
BUGZILLA_COMPONENT_API = "component.get"
PKGDB2_BUGZILLA_NOTIFY_USER = '{{ bugzilla_user }}'
PKGDB2_BUGZILLA_NOTIFY_PASSWORD = '{{ bugzilla_password }}'
PKGDB2_BUGZILLA_DRY_RUN = False
### FAS information
## URL to the FAS instance to query
{% if env == 'staging' %}
PKGDB2_FAS_URL = 'https://admin.stg.fedoraproject.org/accounts'
PKGDB2_FAS_INSECURE = True
SITE_ROOT = 'https://admin.stg.fedoraproject.org'
SITE_URL = '%s/pkgdb' % SITE_ROOT
## Upon changes in pkgdb, update bugzilla
PKGDB2_BUGZILLA_NOTIFICATION = False
FAS_OPENID_ENDPOINT = 'https://id.stg.fedoraproject.org/openid/'
{% else %}
PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts'
SITE_ROOT = 'https://admin.fedoraproject.org'
SITE_URL = '%s/pkgdb' % SITE_ROOT
## Upon changes in pkgdb, update bugzilla
PKGDB2_BUGZILLA_NOTIFICATION = True
FAS_OPENID_ENDPOINT = 'https://id.fedoraproject.org/openid/'
{% endif %}
## name of the user the pkgdb application can log in to FAS with
PKGDB2_FAS_USER = '{{ fedorathirdpartyUser }}'
## password of the user the pkgdb application can log in to FAS with
PKGDB2_FAS_PASSWORD = '{{ fedorathirdpartyPassword }}'
### pkgdb notifications
## Pkgdb sends its notifications by email
PKGDB2_EMAIL_NOTIFICATION = False
## Pkgdb broadcasts its notifications via fedmsg
PKGDB2_FEDMSG_NOTIFICATION = True
## Template to build the email address pkgdb sends its notifications to
PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org'
## The From address email notifications are sent with
PKGDB2_EMAIL_FROM = 'pkgdb@fedoraproject.org'
## The SMTP server to use to send email notifications
PKGDB2_EMAIL_SMTP_SERVER = 'localhost'
PKGDB2_EMAIL_CC = 'scm-commits@lists.fedoraproject.org'
### Email stacktrace
## pkgdb sends email when it faces an exception (trying to add an existing
## package or something alike. These emails are sent to the address set
## here:
MAIL_ADMIN = 'pingou@fedoraproject.org'
# This is required to fix login
PREFERRED_URL_SCHEME='https'
# Make browsers send session cookie only via HTTPS
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_PATH = '/pkgdb/'
SESSION_COOKIE_NAME = 'pkgdb'
# Used by SESSION_COOKIE_PATH
APPLICATION_ROOT = '/pkgdb/'
# PkgDB sync bugzilla email
PKGDB_SYNC_BUGZILLA_EMAIL = """Greetings.
You are receiving this email because there's a problem with your
bugzilla.redhat.com account.
If you recently changed the email address associated with your
Fedora account in the Fedora Account System, it is now out of sync
with your bugzilla.redhat.com account. This leads to problems
with Fedora packages you own or are CC'ed on bug reports for.
Please take one of the following actions:
a) login to your old bugzilla.redhat.com account and change the email
address to match your current email in the Fedora account system.
https://bugzilla.redhat.com login, click preferences, account
information and enter new email address.
b) Create a new account in bugzilla.redhat.com to match your
email listed in your Fedora account system account.
https://bugzilla.redhat.com/ click 'new account' and enter email
address.
c) Change your Fedora Account System email to match your existing
bugzilla.redhat.com account.
https://admin.fedoraproject.org/accounts login, click on 'my account',
then 'edit' and change your email address.
If you have questions or concerns, please let us know.
Your prompt attention in this matter is appreciated.
The Fedora admins.
"""
{% if env != 'staging' %}
FEDOCAL_URL = 'https://apps.fedoraproject.org/calendar'
FEDOCAL_CALENDAR_SHIELD = 'vacation'
REPO_MAP = [
('rawhide', 'fedora/linux/development/rawhide/Everything/source/tree'),
('f24_rel', 'fedora/linux/releases/24/Everything/source/tree'),
('f25_rel', 'fedora/linux/releases/25/Everything/source/tree'),
('f24_up', 'fedora/linux/updates/24/SRPMS/'),
('f25_up', 'fedora/linux/updates/25/SRPMS/'),
('el7', 'epel/7/SRPMS/'),
('el6', 'epel/6/SRPMS/'),
('el5', 'epel/5/SRPMS/'),
]
BASE_REPO_URL = 'http://dl.phx2.fedoraproject.org/pub/%s/'
{% else %}
# URLs used in the package's info page
# Watch for the `%s` in the URL it is mandatory and in each of these, it
# will be replaced by the package's name
PKGS_BUG_URL = 'https://apps.stg.fedoraproject.org/packages/%s/bugs'
PKGS_PKG_URL = 'https://apps.stg.fedoraproject.org/packages/%s'
CGIT_URL = 'http://pkgs.stg.fedoraproject.org/cgit/%s/%s.git/'
BODHI_URL = 'https://bodhi.stg.fedoraproject.org/updates/?packages=%s'
KOJI_URL = 'http://koji.stg.fedoraproject.org/koji/search?'\
'type=package&match=glob&terms=%s'
{% endif %}

23
roles/pkgdb2/templates/pkgdb2.conf
View file

@ -1,23 +0,0 @@
Alias /pkgdb/static /usr/lib/python2.7/site-packages/pkgdb2/static/
WSGIDaemonProcess pkgdb user=apache maximum-requests=1000 display-name=pkgdb processes={{ wsgi_procs }} threads={{ wsgi_threads }}
WSGISocketPrefix run/wsgi
WSGIRestrictStdout On
WSGIRestrictSignal Off
WSGIPythonOptimize 1
WSGIScriptAlias /pkgdb /var/www/pkgdb2.wsgi
<Location />
WSGIProcessGroup pkgdb
<IfModule mod_authz_core.c>
# Apache 2.4
Require all granted
</IfModule>
<IfModule !mod_authz_core.c>
# Apache 2.2
Order deny,allow
Allow from all
</IfModule>
</Location>

22
roles/pkgdb2/templates/pkgdb2.wsgi
View file

@ -1,22 +0,0 @@
#-*- coding: UTF-8 -*-
# The three lines below are required to run on EL6 as EL6 has
# two possible version of python-sqlalchemy and python-jinja2
# These lines make sure the application uses the correct version.
import __main__
__main__.__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4']
import pkg_resources
import os
## Set the environment variable pointing to the configuration file
os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg'
## The following is only needed if you did not install pkgdb
## as a python module (for example if you run it from a git clone).
#import sys
#sys.path.insert(0, '/path/to/pkgdb/')
## The most import line to make the wsgi working
from pkgdb2 import APP as application

78
roles/pkgdb2/templates/pkgdb2_admin.cfg
View file

@ -1,78 +0,0 @@
# Beware that the quotes around the values are mandatory
### Secret key for the Flask application
SECRET_KEY='{{ pkgdb2_secret_key }}'
### url to the database server:
#DB_URL=mysql://user:pass@host/db_name
#DB_URL=postgres://user:pass@host/db_name
DB_URL='postgresql://{{ pkgdb2_db_admin_user }}:{{ pkgdb2_db_admin_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}'
### the number of items (packages, packagers..) to display on the search
### pages
ITEMS_PER_PAGE = 50
### List the ACL which are automatically approved (don't need reviewing)
AUTO_APPROVE = ['watchcommits', 'watchbugzilla']
#### FAS group for the pkgdb admins
ADMIN_GROUP = ['sysadmin-main', 'sysadmin-cvs']
### The default backend for dogpile
### Options are listed at:
### http://dogpilecache.readthedocs.org/en/latest/api.html (backend section)
PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached'
PKGDB2_CACHE_KWARGS = {
'arguments': {
'url': "127.0.0.1:11211",
}
}
### Bugzilla information
## Upon changes in pkgdb, update bugzilla
PKGDB2_BUGZILLA_NOTIFICATION = False
## URL to the bugzilla instance to update
PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com'
## name of the user the pkgdb application can log in to bugzilla with
PKGDB2_BUGZILLA_USER = None
## password of the user the pkgdb application can log in to bugzilla with
PKGDB2_BUGZILLA_PASSWORD = None
### FAS information
## URL to the FAS instance to query
{% if env == 'staging' %}
PKGDB2_FAS_URL = 'https://admin.stg.fedoraproject.org/accounts'
{% else %}
PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts'
{% endif %}
## name of the user the pkgdb application can log in to FAS with
PKGDB2_FAS_USER = '{{ fedorathirdpartyUser }}'
## password of the user the pkgdb application can log in to FAS with
PKGDB2_FAS_PASSWORD = '{{ fedorathirdpartyPassword }}'
### pkgdb notifications
## Pkgdb broadcasts its notifications via fedmsg
PKGDB2_FEDMSG_NOTIFICATION = True
## Pkgdb sends its notifications by email
PKGDB2_EMAIL_NOTIFICATION = False
## Template to build the email address pkgdb sends its notifications to
PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org'
## The From address email notifications are sent with
PKGDB2_EMAIL_FROM = 'nobody@fedoraproject.org'
## The SMTP server to use to send email notifications
PKGDB2_EMAIL_SMTP_SERVER = 'localhost'
### Email stacktrace
## pkgdb sends email when it faces an exception (trying to add an existing
## package or something alike. These emails are sent to the address set
## here:
MAIL_ADMIN = 'pingou@fedoraproject.org'

7
roles/pkgdb2/templates/pkgdb_update_package_info.cron
View file

@ -1,7 +0,0 @@
# Cron updating weekly the pkgdb database to adjust summary and description of
# each package.
#
MAILTO=root
0 10 * * 5 root PKGDB2_CONFIG=/etc/pkgdb2/pkgdb2.cfg /usr/bin/update_package_info.py

2
roles/plus-plus-service/templates/settings.py
View file

@ -11,8 +11,6 @@ PLUS_PLUS_TOKEN = '{{ plus_plus_service_token }}'
{% if env == 'staging' %}
FAS_URL = 'https://admin.stg.fedoraproject.org/accounts/'
PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb/'
{% else %}
FAS_URL = 'https://admin.fedoraproject.org/accounts/'
PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb/'
{% endif %}

1
roles/taskotron/taskotron-trigger/defaults/main.yml
View file

@ -1,7 +1,6 @@
---
trigger_joblog_file: /var/log/taskotron-trigger/jobs.csv
trigger_critpath_file: /var/lib/taskotron-trigger/critpath_whitelist
trigger_critpath_url: https://admin.fedoraproject.org/pkgdb/api/critpath?format=json
trigger_cache_dir: /var/lib/taskotron-trigger/cache
trigger_rules_template_path: /etc/taskotron/trigger_rules.yml
extra_enablerepos: ''

22
roles/varnish/templates/proxies.vcl.j2
View file

@ -13,12 +13,6 @@ backend mirrorlists {
.port = "10002";
}
backend pkgdb {
.host = "localhost";
.port = "10003";
.first_byte_timeout = 160s;
}
backend mailman {
.host = "mailman01";
.port = "http";
@ -214,14 +208,6 @@ sub vcl_recv {
if (req.url ~ "^/mirrorlist/") {
set req.backend_hint = mirrorlists;
}
if (req.url ~ "^/pkgdb") {
set req.backend_hint = pkgdb;
if (req.url ~ "^/pkgdb/static/") {
unset req.http.cookie;
set req.http.clear-cookies = "yes";
set req.url = regsub(req.url, "\?.*", "");
}
}
if (req.url ~ "^/accounts/") {
set req.backend_hint = fas.backend();
if (req.url ~ "^/accounts/static/") {
@ -387,14 +373,6 @@ sub vcl_recv {
# }
}
# When requesting application icons, don't allow cherrypy to set cookies
#sub vcl_backend_fetch {
# if (req.url ~ "^/pkgdb/appicon/show/") {
# unset beresp.http.set-cookie;
# }
#}
# Make sure mirrormanager/mirrors doesn't set any cookies
# (Setting cookies would make varnish store a HIT-FOR-PASS
# making it always fetch from backend)