From 6b0f5d1676cf379fca92d3810e77f19bf3233eaa Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 17 Nov 2015 16:53:19 +0000 Subject: [PATCH] Take a stab at a rewrite of the packages role. --- playbooks/manual/rebuild/fedora-packages.yml | 4 - .../packages3/web/files/cron-flush-pkgs-cache | 1 - .../web/files/cron-sync-latest-builds | 1 - .../web/files/cron-sync-package-index | 2 - roles/packages3/web/files/cron-sync-yum | 1 - .../packages3/web/files/cron_fcomm_log_rotate | 24 -- roles/packages3/web/files/packages-yum.conf | 269 ------------------ roles/packages3/web/files/sync-yum | 34 --- roles/packages3/web/tasks/main.yml | 62 ++-- .../web/templates/fedoracommunity.py | 7 + 10 files changed, 24 insertions(+), 381 deletions(-) delete mode 100644 roles/packages3/web/files/cron-flush-pkgs-cache delete mode 100644 roles/packages3/web/files/cron-sync-latest-builds delete mode 100644 roles/packages3/web/files/cron-sync-package-index delete mode 100644 roles/packages3/web/files/cron-sync-yum delete mode 100644 roles/packages3/web/files/cron_fcomm_log_rotate delete mode 100644 roles/packages3/web/files/packages-yum.conf delete mode 100644 roles/packages3/web/files/sync-yum create mode 100644 roles/packages3/web/templates/fedoracommunity.py diff --git a/playbooks/manual/rebuild/fedora-packages.yml b/playbooks/manual/rebuild/fedora-packages.yml index 5c53a4bc19..79f56ccb8f 100644 --- a/playbooks/manual/rebuild/fedora-packages.yml +++ b/playbooks/manual/rebuild/fedora-packages.yml @@ -51,14 +51,10 @@ - packages/icons - packages/xapian - packages/xapian/search - - packages/xapian/versionmap - packages/tmp - packages/tmp/icons - - packages/tmp/rpms - packages/tmp/search - - packages/tmp/versionmap - packages/tmp/var - - packages/tmp/yum-cache when: install_packages_indexer - name: Rebuild that search idex that we just destroyed diff --git a/roles/packages3/web/files/cron-flush-pkgs-cache b/roles/packages3/web/files/cron-flush-pkgs-cache deleted file mode 100644 index a4e1f0aee5..0000000000 --- a/roles/packages3/web/files/cron-flush-pkgs-cache +++ /dev/null @@ -1 +0,0 @@ -0 2 * * * apache /usr/bin/find /var/cache/fedoracommunity/git.fedoraproject.org -mindepth 1 -maxdepth 1 -type d -exec rm -rf {} \; >> /var/log/fedoracommunity/cron-flush-pkgs-cache.log 2>&1 diff --git a/roles/packages3/web/files/cron-sync-latest-builds b/roles/packages3/web/files/cron-sync-latest-builds deleted file mode 100644 index e97b6d35fa..0000000000 --- a/roles/packages3/web/files/cron-sync-latest-builds +++ /dev/null @@ -1 +0,0 @@ -*/15 * * * * apache /usr/bin/fcomm-index-latest-builds -p /var/cache/fedoracommunity/packages/xapian --koji-url http://koji.fedoraproject.org/kojihub >> /var/log/fedoracommunity/cron-sync-latest-builds.log 2>&1 diff --git a/roles/packages3/web/files/cron-sync-package-index b/roles/packages3/web/files/cron-sync-package-index deleted file mode 100644 index 8b76c84dc0..0000000000 --- a/roles/packages3/web/files/cron-sync-package-index +++ /dev/null @@ -1,2 +0,0 @@ -0 0 * * * apache /usr/bin/fcomm-index-packages -p /var/cache/fedoracommunity/packages/tmp -y /etc/fedoracommunity/yum-repo.conf --index-db-dest /var/cache/fedoracommunity/packages/xapian --icons-dest /var/cache/fedoracommunity/packages/icons --tagger-url https://apps.fedoraproject.org/tagger/api/v1/tag/export/ --pkgdb-url https://admin.fedoraproject.org/pkgdb >> /var/log/fedoracommunity/cron-sync-package-index.log 2>&1 - diff --git a/roles/packages3/web/files/cron-sync-yum b/roles/packages3/web/files/cron-sync-yum deleted file mode 100644 index b5080e0ef1..0000000000 --- a/roles/packages3/web/files/cron-sync-yum +++ /dev/null @@ -1 +0,0 @@ -*/15 * * * * apache /usr/local/bin/sync-yum >> /var/log/fedoracommunity/cron-sync-yum.log 2>&1 diff --git a/roles/packages3/web/files/cron_fcomm_log_rotate b/roles/packages3/web/files/cron_fcomm_log_rotate deleted file mode 100644 index cd2e6244ff..0000000000 --- a/roles/packages3/web/files/cron_fcomm_log_rotate +++ /dev/null @@ -1,24 +0,0 @@ -/var/log/fedoracommunity/cron-sync-latest-builds.log { - missingok - notifempty - rotate 4 - weekly -} -/var/log/fedoracommunity/cron-sync-package-builds.log { - missingok - notifempty - rotate 4 - weekly -} -/var/log/fedoracommunity/cron-sync-yum.log { - missingok - notifempty - rotate 4 - weekly -} -/var/log/fedoracommunity/cron-flush-pkgs-cache.log { - missingok - notifempty - rotate 4 - weekly -} diff --git a/roles/packages3/web/files/packages-yum.conf b/roles/packages3/web/files/packages-yum.conf deleted file mode 100644 index 6684fb2c57..0000000000 --- a/roles/packages3/web/files/packages-yum.conf +++ /dev/null @@ -1,269 +0,0 @@ -[main] -cachedir=/var/cache/fedoracommunity/packages/tmp/ -installroot=/var/cache/fedoracommunity/packages/tmp/ -keepcache=1 -debuglevel=2 -logfile=yum.log -exactarch=0 -obsoletes=1 -gpgcheck=1 -plugins=0 -installonly_limit=3 -reposdir= -cacheonly=1 - -[rawhide-x86_64] -name=Fedora - Rawhide - Developmental packages for the next Fedora release -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/x86_64/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64 -enabled=0 -gpgcheck=0 - -[rawhide-i686] -name=Fedora - Rawhide - Developmental packages for the next Fedora release -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/i386/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=i386 -enabled=0 -gpgcheck=0 - -[rawhide-debuginfo-x86_64] -name=Fedora - Rawhide - Debug -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/x86_64/debug/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide-debug&arch=x86_64 -enabled=0 -gpgcheck=0 - -[rawhide-source] -name=Fedora - Rawhide - Source -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/development/rawhide/source/SRPMS/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=rawhide-source&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-23-x86_64] -name=Fedora 23 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/23/Everything/x86_64/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-23&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-23-updates-x86_64] -name=Fedora 23 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/23/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f23&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-23-testing-x86_64] -name=Fedora 23 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/23/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f23&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-23-i686] -name=Fedora 23 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/23/Everything/i386/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-23&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-23-updates-i686] -name=Fedora 23 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/23/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f23&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-23-testing-i686] -name=Fedora 23 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/23/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f23&arch=i386 -enabled=0 - - -[fedora-22-x86_64] -name=Fedora 22 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/x86_64/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-22-updates-x86_64] -name=Fedora 22 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-22-testing-x86_64] -name=Fedora 22 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-22-i686] -name=Fedora 22 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/22/Everything/i386/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-22&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-22-updates-i686] -name=Fedora 22 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/22/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f22&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-22-testing-i686] -name=Fedora 22 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/22/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f22&arch=i386 -enabled=0 - - - -[fedora-21-x86_64] -name=Fedora 21 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/21/Everything/x86_64/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-21&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-21-updates-x86_64] -name=Fedora 21 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/21/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f21&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-21-testing-x86_64] -name=Fedora 21 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/21/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f21&arch=x86_64 -enabled=0 -gpgcheck=0 - -[fedora-21-i686] -name=Fedora 21 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/releases/21/Everything/i386/os/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=fedora-21&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-21-updates-i686] -name=Fedora 21 - Updates -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/21/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-released-f21&arch=i386 -enabled=0 -gpgcheck=0 - -[fedora-21-testing-i686] -name=Fedora 21 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/21/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=updates-testing-f21&arch=i386 -enabled=0 - -[epel-5-x86_64] -name=EPEL 5 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/5/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-5&arch=x86_64 -enabled=0 -gpgcheck=0 - -[epel-5-testing-x86_64] -name=EPEL 5 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/5/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel5&arch=x86_64 -enabled=0 -gpgcheck=0 - -[epel-5-i686] -name=EPEL 5 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/5/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-5&arch=i386 -enabled=0 -gpgcheck=0 - -[epel-5-testing-i686] -name=EPEL 5 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/5/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel5&arch=i386 -enabled=0 -gpgcheck=0 - -[epel-6-x86_64] -name=EPEL 6 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/6/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=x86_64 -enabled=0 -gpgcheck=0 - -[epel-6-testing-x86_64] -name=EPEL 6 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/6/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel6&arch=x86_64 -enabled=0 -gpgcheck=0 - -[epel-6-i686] -name=EPEL 6 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/6/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=i386 -enabled=0 -gpgcheck=0 - -[epel-6-testing-i686] -name=EPEL 6 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/6/i386/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel6&arch=i386 -enabled=0 -gpgcheck=0 - -[epel-7-x86_64] -name=EPEL 7 -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/7/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64 -enabled=0 -gpgcheck=0 - -[epel-7-testing-x86_64] -name=EPEL 7 - Testing -failovermethod=priority -baseurl=http://download01.phx2.fedoraproject.org/pub/epel/testing/7/x86_64/ -#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=testing-epel7&arch=x86_64 -enabled=0 -gpgcheck=0 diff --git a/roles/packages3/web/files/sync-yum b/roles/packages3/web/files/sync-yum deleted file mode 100644 index afd5ff75b9..0000000000 --- a/roles/packages3/web/files/sync-yum +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -LOCKFILE=/var/cache/fedoracommunity/fedora-packages-yum.lock - -rebuild=$(cat <<"EOF" -echo 'Trying makecache.' -/usr/bin/yum makecache -c /etc/fedoracommunity/yum-repo.conf --enablerepo=* -echo 'Releasing $LOCKFILE' -EOF -) -nuke_and_rebuild=$(cat <<"EOF" -echo 'Nuking.' -rm /var/cache/fedoracommunity/packages/tmp/var/lib/rpm/__db.* -/usr/bin/rpmdb --root=/var/cache/fedoracommunity/packages/tmp/var/lib/rpm --rebuilddb -echo 'Trying makecache again now.' -/usr/bin/yum makecache -c /etc/fedoracommunity/yum-repo.conf --enablerepo=* -echo 'Releasing $LOCKFILE' -EOF -) - - -echo "Acquiring $LOCKFILE..." -flock $LOCKFILE -c "$rebuild" - - -# If it failed, then try to nuke and rebuild the rpmdb first. -if [ $? -eq 1 ] ; then - echo 'makecache failed... sleeping for 10 seconds.' - sleep 10 - echo 'Waking. Now trying to rebuild the rpmdb.' - - echo "Acquiring $LOCKFILE..." - flock $LOCKFILE -c "$nuke_and_rebuild" -fi diff --git a/roles/packages3/web/tasks/main.yml b/roles/packages3/web/tasks/main.yml index 338ffe5338..5eda2cf65c 100644 --- a/roles/packages3/web/tasks/main.yml +++ b/roles/packages3/web/tasks/main.yml @@ -8,7 +8,6 @@ - python-psycopg2 - python-memcached - python-sqlalchemy0.8 - - redis tags: - packages - packages/web @@ -17,7 +16,6 @@ file: path={{ item }} state=directory owner=apache group=apache mode=755 with_items: - /etc/fedoracommunity - - /etc/fedoracommunity/yum_cache.repos.d - /var/cache/fedoracommunity # the gluster role usually creates this one - /var/log/fedoracommunity tags: @@ -39,7 +37,7 @@ owner=apache group=apache mode=0600 notify: - reload httpd - - restart fcomm-cache-worker + - restart fedmsg-hub tags: - packages - packages/web @@ -55,14 +53,13 @@ - packages - packages/web -- name: Copy over the yum config - copy: > - src=packages-yum.conf - dest=/etc/fedoracommunity/yum-repo.conf - owner=root group=root mode=0644 +- name: Copy over the fedmsg consumer config + template: > + src=fedoracommunity.py + dest=/etc/fedmsg.d/fedoracommunity.py + owner=root group=root mode=644 notify: - - reload httpd - - restart fcomm-cache-worker + - restart fedmsg-hub tags: - packages - packages/web @@ -88,16 +85,7 @@ owner=root group=root mode=0644 notify: - reload httpd - - restart fcomm-cache-worker - tags: - - packages - - packages/web - -- name: start some helper services - service: name="{{item}}" state=started enabled=yes - with_items: - - redis - - fcomm-cache-worker + - restart fedmsg-hub tags: - packages - packages/web @@ -117,14 +105,10 @@ - packages/icons - packages/xapian - packages/xapian/search - - packages/xapian/versionmap - packages/tmp - packages/tmp/icons - - packages/tmp/rpms - packages/tmp/search - - packages/tmp/versionmap - packages/tmp/var - - packages/tmp/yum-cache when: install_packages_indexer tags: - packages @@ -141,29 +125,17 @@ - packages/web - icon -- name: Copy some scripts for /usr/local - copy: src="{{item}}" dest="/usr/local/bin/{{item}}" mode=0755 - with_items: - - sync-yum - when: install_packages_indexer - tags: - - packages - - packages/web - -- name: Copy the indexer cronjobs - copy: src="{{item}}" dest="/etc/cron.d/{{item}}" - with_items: - - cron-sync-yum - - cron-sync-latest-builds - - cron-sync-package-index - - cron-flush-pkgs-cache - when: install_packages_indexer - tags: - - packages - - packages/web +# Our fedmsg updater should handle everything, no more need for cron. +#- name: Copy the indexer cronjobs +# copy: src="{{item}}" dest="/etc/cron.d/{{item}}" +# with_items: +# - cron-sync-package-index +# when: install_packages_indexer +# tags: +# - packages +# - packages/web # Lastly, here's some selinux stuff. - - name: set some selinux booleans seboolean: name={{item}} persistent=yes state=yes with_items: diff --git a/roles/packages3/web/templates/fedoracommunity.py b/roles/packages3/web/templates/fedoracommunity.py new file mode 100644 index 0000000000..4bb5213755 --- /dev/null +++ b/roles/packages3/web/templates/fedoracommunity.py @@ -0,0 +1,7 @@ +config = { + {% if install_packages_indexer %} + 'fedoracommunity.fedmsg.consumer.enabled': True, + {% else %} + 'fedoracommunity.fedmsg.consumer.enabled': False, + {% endif %} +}