diff --git a/handlers/restart_services.yml b/handlers/restart_services.yml index 4eead5b27e..805ee4e11e 100644 --- a/handlers/restart_services.yml +++ b/handlers/restart_services.yml @@ -83,6 +83,12 @@ - name: restart rsyslog action: service name=rsyslog state=restarted +- name: restart sks-db + action: service name=sks-db state=restarted + +- name: restart sks-recon + action: service name=sks-recon state=restarted + - name: restart sshd action: service name=sshd state=restarted diff --git a/inventory/host_vars/keys01.fedoraproject.org b/inventory/host_vars/keys01.fedoraproject.org index e9b26d3dbb..4b08a8b2f1 100644 --- a/inventory/host_vars/keys01.fedoraproject.org +++ b/inventory/host_vars/keys01.fedoraproject.org @@ -1,6 +1,6 @@ --- nm: 255.255.255.240 -gw: 80.239.156.208 +gw: 80.239.156.209 dns: 213.248.76.210 ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-6 ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL6-x86_64/ diff --git a/inventory/host_vars/value03.phx2.fedoraproject.org b/inventory/host_vars/value03.phx2.fedoraproject.org new file mode 100644 index 0000000000..a26aff6ed3 --- /dev/null +++ b/inventory/host_vars/value03.phx2.fedoraproject.org @@ -0,0 +1,3 @@ +--- +host_backup_targets: ['/srv'] +host_backup_targets: ['/var/lib/zodbot'] diff --git a/inventory/inventory b/inventory/inventory index cb8e5c2189..bfdd311095 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -59,6 +59,7 @@ lockbox01.phx2.fedoraproject.org people03.vpn.fedoraproject.org pkgs01.phx2.fedoraproject.org log02.phx2.fedoraproject.org +value03.phx2.fedoraproject.org [badges-backend] badges-backend01.phx2.fedoraproject.org @@ -147,7 +148,6 @@ collab04.fedoraproject.org [releng] releng01.phx2.fedoraproject.org releng02.phx2.fedoraproject.org -releng03.phx2.fedoraproject.org releng04.phx2.fedoraproject.org relepel01.phx2.fedoraproject.org diff --git a/playbooks/hosts/copr-be-dev.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-be-dev.cloud.fedoraproject.org.yml index 0b886559b3..0cf3e3e308 100644 --- a/playbooks/hosts/copr-be-dev.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-be-dev.cloud.fedoraproject.org.yml @@ -141,11 +141,5 @@ tags: - provision_config - - - - - - handlers: - include: $handlers/restart_services.yml diff --git a/playbooks/hosts/copr-be.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-be.cloud.fedoraproject.org.yml index a1c7bd2324..51e24d0e10 100644 --- a/playbooks/hosts/copr-be.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-be.cloud.fedoraproject.org.yml @@ -146,11 +146,5 @@ tags: - provision_config - - - - - - handlers: - include: $handlers/restart_services.yml diff --git a/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml index 2b9551de79..7b0758218b 100644 --- a/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml @@ -53,7 +53,5 @@ - httpd - postgresql - - handlers: - include: $handlers/restart_services.yml diff --git a/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml index a8dcf2c95f..3856880805 100644 --- a/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml @@ -61,7 +61,5 @@ - --service=https - --service=http - - handlers: - include: $handlers/restart_services.yml diff --git a/playbooks/rdiff-backup.yml b/playbooks/rdiff-backup.yml index cf7ff70be9..9b13046c15 100644 --- a/playbooks/rdiff-backup.yml +++ b/playbooks/rdiff-backup.yml @@ -20,11 +20,11 @@ tasks: - name: run rdiff-backup hitting all the global targets - local_action: "shell rdiff-backup --create-full-path --print-statistics ${inventory_hostname}::${item} /fedora_backups/${inventory_hostname}/`basename ${item}`" + local_action: "shell rdiff-backup --create-full-path --print-statistics ${inventory_hostname}::${item} /fedora_backups/${inventory_hostname}/`basename ${item}` | mail -r sysadmin-backup-members@fedoraproject.org -s 'rdiff-backup: ${inventory_hostname}:${item}' sysadmin-backup-members@fedoraproject.org" with_items: $global_backup_targets when: global_backup_targets is defined - name: run rdiff-backup hitting all the host targets - local_action: "shell rdiff-backup --create-full-path --print-statistics ${inventory_hostname}::${item} /fedora_backups/${inventory_hostname}/`basename ${item}`" + local_action: "shell rdiff-backup --create-full-path --print-statistics ${inventory_hostname}::${item} /fedora_backups/${inventory_hostname}/`basename ${item}` | mail -r sysadmin-backup-members@fedoraproject.org -s 'rdiff-backup: ${inventory_hostname}:${item}' sysadmin-backup-members@fedoraproject.org" with_items: $host_backup_targets when: host_backup_targets is defined diff --git a/playbooks/package_update.yml b/playbooks/rkhunter_update.yml similarity index 100% rename from playbooks/package_update.yml rename to playbooks/rkhunter_update.yml diff --git a/playbooks/sign_and_import.yml b/playbooks/sign_and_import.yml new file mode 100644 index 0000000000..5908ebcdc6 --- /dev/null +++ b/playbooks/sign_and_import.yml @@ -0,0 +1,38 @@ +# This playbook takes new rpms specified with a fileglob, signs them, and adds +# them to the infrastructure repo. +# +# requires --extra-vars="rpmdir='/home/fedora/ralph/rpms/'" + +# TODO -- grab rpms from koji build/task ids beforehand? +# TODO -- how do make it easy to select the infra-testing repo? +# TODO -- other arches than x86_64? + +- name: batch sign and import a directory full of rpms + user: root + + tasks: + + - name: sign all the rpms with our gpg key + local_action: /bin/rpm --resign ${rpmdir}/${item}.rpm + with_fileglob: ${rpmdir}/*.rpm + + - name: copy the source rpms to the SRPMS dir + local_action: copy src=${rpmdir}/${item} dest=/mnt/fedora/app/fi-repo/6/SRPMS/${item} + with_fileglob: ${rpmdir}/*.src.rpm + + - name: Run createrepo on the srpm dir + local_action: createrepo --update /mnt/fedora/app/fi-repo/6/SRPMS/ + + - name: copy the binary rpms to the x86_64 dir + local_action: copy src=${rpmdir}/${item} dest=/mnt/fedora/app/fi-repo/6/x86_64/${item} + with_fileglob: ${rpmdir}/*.rpm + + - name: Run createrepo on the x86_64 dir + local_action: createrepo --update /mnt/fedora/app/fi-repo/6/x86_64/ + + - name: make a directory where we store the rpms afterwards + local_action: file path=${rpmdir}-old state=directory + + - name: move processed rpms out to ${rpmdir}-old + local_action: "/bin/mv ${rpmdir}/${item} ${rpmdir}-old/${item}" + with_fileglob: ${rpmdir}/*.rpm diff --git a/playbooks/update_packages.yml b/playbooks/update_packages.yml new file mode 100644 index 0000000000..9bb9f18ada --- /dev/null +++ b/playbooks/update_packages.yml @@ -0,0 +1,24 @@ +# requires --extra-vars="target='host1;host2;group etc' package='python-tahrir' +# +# Alternatively, you could update a group of packages like +# --extra-vars="package='python-t*'" + +# TODO -- how do make it easy to select the infra-testing repo? + +- name: push packages out + hosts: $target + user: root + + tasks: + + # TODO -- I'm not sure if state=latest will go ahead and expire the cache for + # us or not. I'll comment this out for now, but if we experience problems in + # the future we can just uncomment it. It definitely works with it in place, + # the problem is that the expire-cache command is not idempotent which results + # in partly confusing ansible-playbook results. -threebean + # + #- name: expire-caches + # action: command yum clean expire-cache + # + - name: yum update ${package} + yum: name=${package} state=latest diff --git a/roles/badges-backend/files/cron/award-oldschool-badges b/roles/badges-backend/files/cron/award-oldschool-badges new file mode 100644 index 0000000000..f4569a5388 --- /dev/null +++ b/roles/badges-backend/files/cron/award-oldschool-badges @@ -0,0 +1,158 @@ +#!/usr/bin/env python + +import __main__ +# This is going to require sqlalchemy 0.8 sooner than later. +__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; +import pkg_resources +pkg_resources.require(__requires__) + +import time +import urllib +import socket +from hashlib import md5 +import getpass +import pprint + +from tahrir_api.dbapi import TahrirDatabase +import transaction + + +_fas_cache = {} + +import logging +log = logging.getLogger() +logging.basicConfig() +import fedora.client.fas2 + + +import fedmsg +import fedmsg.config + +fm_config = fedmsg.config.load_config() +fm_config['cert_prefix'] = 'fedbadges' +fm_config['name'] = 'relay_inbound' +fm_config['active'] = True +fedmsg.init(**fm_config) + + +def get_cla_signers(**config): + creds = config['fas_credentials'] + + fasclient = fedora.client.fas2.AccountSystem( + username=creds['username'], + password=creds['password'], + ) + + timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(600) + try: + log.info("Downloading FAS cache") + request = fasclient.send_request('/user/list', + req_params={'search': '*'}, + auth=True) + finally: + socket.setdefaulttimeout(timeout) + + mega_list = request['people'] + request['unapproved_people'] + print len(mega_list), "people in total" + print len(request['people']), "'people'" + print len(request['unapproved_people']), "'unapproved'" + + clas, sponsors, provenpackagers, proventesters = [], [], [], [] + for user in mega_list: + if any([group.name == 'cla_done' for group in user.memberships]): + clas.append(user) + if any([group.name == 'provenpackager' for group in user.memberships]): + provenpackagers.append(user) + if any([group.name == 'proventesters' for group in user.memberships]): + proventesters.append(user) + + return clas, sponsors, provenpackagers, proventesters + + +def main(): + d = {} + print "fascache.db code is commented out -- querying fas." + clas, sponsors, provenpackagers, proventesters = \ + get_cla_signers(fas_credentials=fm_config['fas_credentials']) + d['clas'] = clas + d['sponsors'] = sponsors + d['provenpackagers'] = provenpackagers + d['proventesters'] = proventesters + + # When running by hand, its often nice to use the code below instead of the code above. + #import shelve + #d = shelve.open("fascache.db") + #if not d: + # print "fascache.db not found.. querying fas." + # clas, sponsors, provenpackagers, proventesters = \ + # get_cla_signers(fas_credentials=dict( + # username="ralph", + # password=getpass.getpass(), + # )) + # d['clas'] = clas + # d['sponsors'] = sponsors + # d['provenpackagers'] = provenpackagers + # d['proventesters'] = proventesters + # d.sync() + #else: + # print "** using cached fas data **" + + clas, sponsors, provenpackagers, proventesters = \ + d['clas'], d['sponsors'], \ + d['provenpackagers'], d['proventesters'] + + print len(list(clas)), "clas in the end" + print len(list(sponsors)), "sponsors in the end" + print len(list(provenpackagers)), "provenpackagers in the end" + print len(list(proventesters)), "proventesters in the end" + + + print "Awarding for involvement." + badge = tahrir.get_badge(badge_id='involvement') + hit_em_up(badge, clas) + + badge = tahrir.get_badge(badge_id='proven-packager') + hit_em_up(badge, provenpackagers) + + badge = tahrir.get_badge(badge_id='proven-tester') + hit_em_up(badge, proventesters) + +def hit_em_up(badge, group): + for fas_user in group: + email = fas_user.username + "@fedoraproject.org" + user = tahrir.get_person(email) + + if not user: + continue + + if tahrir.assertion_exists(badge.id, email): + print email, "already has", badge.id, "skipping." + continue + + time.sleep(1) + print "awarding", badge.id, "to", email + try: + transaction.begin() + tahrir.add_assertion(badge.id, email, None) + transaction.commit() + fedmsg.publish(topic="badge.award", + modname="fedbadges", + msg=dict( + badge=dict( + name=badge.name, + description=badge.description, + image_url=badge.image, + ), + user=dict( + username=user.nickname, + badges_user_id=user.id, + ), + )) + except Exception as e: + transaction.abort() + print "Failure:", e + +uri = fm_config['badges_global']['database_uri'] +tahrir = TahrirDatabase(uri) +main() diff --git a/roles/badges-backend/files/cron/award-oldschool-badges.cron b/roles/badges-backend/files/cron/award-oldschool-badges.cron new file mode 100644 index 0000000000..fd7b3ce7b4 --- /dev/null +++ b/roles/badges-backend/files/cron/award-oldschool-badges.cron @@ -0,0 +1 @@ +*/25 * * * * fedmsg /usr/share/badges/cronjobs/award-oldschool-badges diff --git a/roles/badges-backend/tasks/main.yml b/roles/badges-backend/tasks/main.yml index 14cf0a861c..5fa0624ded 100644 --- a/roles/badges-backend/tasks/main.yml +++ b/roles/badges-backend/tasks/main.yml @@ -54,3 +54,41 @@ - badges notify: - restart fedmsg-hub + + +- name: ensure badges cron directories exist + file: > + state=directory + path=$item + mode=755 + owner=root + with_items: + - /usr/share/badges/cronjobs/ + - /etc/cron.d/ + tags: + - config + - cron + +- name: oldschool badge award scripts + copy: > + src=cron/$item + dest=/usr/share/badges/cronjobs/$item + owner=fedmsg + mode=744 + with_items: + - award-oldschool-badges + tags: + - config + - cron + +- name: oldschool badge award cronjobs + copy: > + src=cron/$item + dest=/etc/cron.d/$item + owner=root + mode=644 + with_items: + - award-oldschool-badges + tags: + - config + - cron diff --git a/roles/badges-frontend/files/tahrir.conf b/roles/badges-frontend/files/tahrir.conf index d5bc4622c4..b65d7041d0 100644 --- a/roles/badges-frontend/files/tahrir.conf +++ b/roles/badges-frontend/files/tahrir.conf @@ -1,5 +1,5 @@ -# We serve static resources dynamically for now. -#Alias /static /usr/share/tahrir/static +Alias /static /usr/lib/python2.6/site-packages/tahrir/static +Alias /pngs /usr/share/badges/pngs WSGIDaemonProcess tahrir user=tahrir group=tahrir maximum-requests=1000 display-name=tahrir processes=4 threads=4 WSGISocketPrefix run/wsgi