Take a stab at a rewrite of the packages role.

playbooks/manual/rebuild/fedora-packages.yml

@@ -51,14 +51,10 @@
     - packages/icons
     - packages/xapian
     - packages/xapian/search
-    - packages/xapian/versionmap
     - packages/tmp
     - packages/tmp/icons
-    - packages/tmp/rpms
     - packages/tmp/search
-    - packages/tmp/versionmap
     - packages/tmp/var
-    - packages/tmp/yum-cache
     when: install_packages_indexer
 
 - name: Rebuild that search idex that we just destroyed

roles/packages3/web/files/cron-flush-pkgs-cache

@@ -1 +0,0 @@
-0 2 * * * apache /usr/bin/find /var/cache/fedoracommunity/git.fedoraproject.org -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \; >> /var/log/fedoracommunity/cron-flush-pkgs-cache.log 2>&1

roles/packages3/web/files/cron-sync-latest-builds

@@ -1 +0,0 @@
-*/15 * * * * apache /usr/bin/fcomm-index-latest-builds -p /var/cache/fedoracommunity/packages/xapian --koji-url http://koji.fedoraproject.org/kojihub >> /var/log/fedoracommunity/cron-sync-latest-builds.log 2>&1

roles/packages3/web/files/cron-sync-package-index

@@ -1,2 +0,0 @@
-0 0 * * * apache /usr/bin/fcomm-index-packages -p /var/cache/fedoracommunity/packages/tmp -y /etc/fedoracommunity/yum-repo.conf --index-db-dest /var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --tagger-url https://apps.fedoraproject.org/tagger/api/v1/tag/export/ --pkgdb-url https://admin.fedoraproject.org/pkgdb >> /var/log/fedoracommunity/cron-sync-package-index.log 2>&1
-

roles/packages3/web/files/cron-sync-yum

@@ -1 +0,0 @@
-*/15 * * * * apache /usr/local/bin/sync-yum >> /var/log/fedoracommunity/cron-sync-yum.log 2>&1

roles/packages3/web/files/cron_fcomm_log_rotate

@@ -1,24 +0,0 @@
-/var/log/fedoracommunity/cron-sync-latest-builds.log {
-   missingok
-   notifempty
-   rotate 4
-   weekly
-}
-/var/log/fedoracommunity/cron-sync-package-builds.log {
-   missingok
-   notifempty
-   rotate 4
-   weekly
-}
-/var/log/fedoracommunity/cron-sync-yum.log {
-   missingok
-   notifempty
-   rotate 4
-   weekly
-}
-/var/log/fedoracommunity/cron-flush-pkgs-cache.log {
-   missingok
-   notifempty
-   rotate 4
-   weekly
-}

roles/packages3/web/files/packages-yum.conf

@@ -1,269 +0,0 @@
-[main]
-cachedir=/var/cache/fedoracommunity/packages/tmp/
-installroot=/var/cache/fedoracommunity/packages/tmp/
-keepcache=1
-debuglevel=2
-logfile=yum.log
-exactarch=0
-obsoletes=1
-gpgcheck=1
-plugins=0
-installonly_limit=3
-reposdir=
-cacheonly=1
-
-[rawhide-x86_64]
-name=Fedora - Rawhide - Developmental packages for the next Fedora release
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/x86_64/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[rawhide-i686]
-name=Fedora - Rawhide - Developmental packages for the next Fedora release
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/i386/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=i386
-enabled=0
-gpgcheck=0
-
-[rawhide-debuginfo-x86_64]
-name=Fedora - Rawhide - Debug
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/x86_64/debug/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide-debug&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[rawhide-source]
-name=Fedora - Rawhide - Source
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/source/SRPMS/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide-source&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-23-x86_64]
-name=Fedora 23
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/23/Everything/x86_64/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-23&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-23-updates-x86_64]
-name=Fedora 23 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/23/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f23&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-23-testing-x86_64]
-name=Fedora 23 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/23/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f23&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-23-i686]
-name=Fedora 23
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/23/Everything/i386/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-23&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-23-updates-i686]
-name=Fedora 23 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/23/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f23&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-23-testing-i686]
-name=Fedora 23 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/23/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f23&arch=i386
-enabled=0
-
-
-[fedora-22-x86_64]
-name=Fedora 22
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/x86_64/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-22-updates-x86_64]
-name=Fedora 22 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-22-testing-x86_64]
-name=Fedora 22 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-22-i686]
-name=Fedora 22
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/i386/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-22-updates-i686]
-name=Fedora 22 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-22-testing-i686]
-name=Fedora 22 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=i386
-enabled=0
-
-
-
-[fedora-21-x86_64]
-name=Fedora 21
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/21/Everything/x86_64/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-21&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-21-updates-x86_64]
-name=Fedora 21 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/21/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f21&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-21-testing-x86_64]
-name=Fedora 21 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/21/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f21&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[fedora-21-i686]
-name=Fedora 21
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/21/Everything/i386/os/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-21&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-21-updates-i686]
-name=Fedora 21 - Updates
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/21/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f21&arch=i386
-enabled=0
-gpgcheck=0
-
-[fedora-21-testing-i686]
-name=Fedora 21 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/21/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f21&arch=i386
-enabled=0
-
-[epel-5-x86_64]
-name=EPEL 5
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/5/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-5&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[epel-5-testing-x86_64]
-name=EPEL 5 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/5/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel5&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[epel-5-i686]
-name=EPEL 5
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/5/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-5&arch=i386
-enabled=0
-gpgcheck=0
-
-[epel-5-testing-i686]
-name=EPEL 5 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/5/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel5&arch=i386
-enabled=0
-gpgcheck=0
-
-[epel-6-x86_64]
-name=EPEL 6
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/6/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[epel-6-testing-x86_64]
-name=EPEL 6 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/6/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel6&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[epel-6-i686]
-name=EPEL 6
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/6/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=i386
-enabled=0
-gpgcheck=0
-
-[epel-6-testing-i686]
-name=EPEL 6 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/6/i386/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel6&arch=i386
-enabled=0
-gpgcheck=0
-
-[epel-7-x86_64]
-name=EPEL 7
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/7/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64
-enabled=0
-gpgcheck=0
-
-[epel-7-testing-x86_64]
-name=EPEL 7 - Testing
-failovermethod=priority
-baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/7/x86_64/
-#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel7&arch=x86_64
-enabled=0
-gpgcheck=0

roles/packages3/web/files/sync-yum

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-LOCKFILE=/var/cache/fedoracommunity/fedora-packages-yum.lock
-
-rebuild=$(cat <<"EOF"
-echo 'Trying makecache.'
-/usr/bin/yum makecache -c /etc/fedoracommunity/yum-repo.conf --enablerepo=*
-echo 'Releasing $LOCKFILE'
-EOF
-)
-nuke_and_rebuild=$(cat <<"EOF"
-echo 'Nuking.'
-rm /var/cache/fedoracommunity/packages/tmp/var/lib/rpm/__db.*
-/usr/bin/rpmdb --root=/var/cache/fedoracommunity/packages/tmp/var/lib/rpm --rebuilddb
-echo 'Trying makecache again now.'
-/usr/bin/yum makecache -c /etc/fedoracommunity/yum-repo.conf --enablerepo=*
-echo 'Releasing $LOCKFILE'
-EOF
-)
-
-
-echo "Acquiring $LOCKFILE..."
-flock $LOCKFILE -c "$rebuild"
-
-
-# If it failed, then try to nuke and rebuild the rpmdb first.
-if [ $? -eq 1 ] ; then
-    echo 'makecache failed... sleeping for 10 seconds.'
-    sleep 10
-    echo 'Waking.  Now trying to rebuild the rpmdb.'
-
-    echo "Acquiring $LOCKFILE..."
-    flock $LOCKFILE -c "$nuke_and_rebuild"
-fi

roles/packages3/web/tasks/main.yml

@@ -8,7 +8,6 @@
   - python-psycopg2
   - python-memcached
   - python-sqlalchemy0.8
-  - redis
   tags:
   - packages
   - packages/web
@@ -17,7 +16,6 @@
   file: path={{ item }} state=directory owner=apache group=apache mode=755
   with_items:
   - /etc/fedoracommunity
-  - /etc/fedoracommunity/yum_cache.repos.d
   - /var/cache/fedoracommunity  # the gluster role usually creates this one
   - /var/log/fedoracommunity
   tags:
@@ -39,7 +37,7 @@
     owner=apache group=apache mode=0600
   notify:
   - reload httpd
-  - restart fcomm-cache-worker
+  - restart fedmsg-hub
   tags:
   - packages
   - packages/web
@@ -55,14 +53,13 @@
   - packages
   - packages/web
 
-- name: Copy over the yum config
+- name: Copy over the fedmsg consumer config
-  copy: >
+  template: >
-    src=packages-yum.conf
+    src=fedoracommunity.py
-    dest=/etc/fedoracommunity/yum-repo.conf
+    dest=/etc/fedmsg.d/fedoracommunity.py
-    owner=root group=root mode=0644
+    owner=root group=root mode=644
   notify:
-  - reload httpd
+  - restart fedmsg-hub
-  - restart fcomm-cache-worker
   tags:
   - packages
   - packages/web
@@ -88,16 +85,7 @@
     owner=root group=root mode=0644
   notify:
   - reload httpd
-  - restart fcomm-cache-worker
+  - restart fedmsg-hub
-  tags:
-  - packages
-  - packages/web
-
-- name: start some helper services
-  service: name="{{item}}" state=started enabled=yes
-  with_items:
-  - redis
-  - fcomm-cache-worker
   tags:
   - packages
   - packages/web
@@ -117,14 +105,10 @@
   - packages/icons
   - packages/xapian
   - packages/xapian/search
-  - packages/xapian/versionmap
   - packages/tmp
   - packages/tmp/icons
-  - packages/tmp/rpms
   - packages/tmp/search
-  - packages/tmp/versionmap
   - packages/tmp/var
-  - packages/tmp/yum-cache
   when: install_packages_indexer
   tags:
   - packages
@@ -141,29 +125,17 @@
   - packages/web
   - icon
 
-- name: Copy some scripts for /usr/local
+# Our fedmsg updater should handle everything, no more need for cron.
-  copy: src="{{item}}" dest="/usr/local/bin/{{item}}" mode=0755
+#- name: Copy the indexer cronjobs
-  with_items:
+#  copy: src="{{item}}" dest="/etc/cron.d/{{item}}"
-  - sync-yum
+#  with_items:
-  when: install_packages_indexer
+#  - cron-sync-package-index
-  tags:
+#  when: install_packages_indexer
-  - packages
+#  tags:
-  - packages/web
+#  - packages
-
+#  - packages/web
-- name: Copy the indexer cronjobs
-  copy: src="{{item}}" dest="/etc/cron.d/{{item}}"
-  with_items:
-  - cron-sync-yum
-  - cron-sync-latest-builds
-  - cron-sync-package-index
-  - cron-flush-pkgs-cache
-  when: install_packages_indexer
-  tags:
-  - packages
-  - packages/web
 
 # Lastly, here's some selinux stuff.
-
 - name: set some selinux booleans
   seboolean: name={{item}} persistent=yes state=yes
   with_items:

roles/packages3/web/templates/fedoracommunity.py

@@ -0,0 +1,7 @@
+config = {
+    {% if install_packages_indexer %}
+    'fedoracommunity.fedmsg.consumer.enabled': True,
+    {% else %}
+    'fedoracommunity.fedmsg.consumer.enabled': False,
+    {% endif %}
+}