Switch mirrormanager to the Openshift version

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

main.yml

@@ -42,7 +42,6 @@
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/maintainer-test.yml
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/mariadb-server.yml
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/memcached.yml
-- import_playbook: /srv/web/infra/ansible/playbooks/groups/mirrormanager.yml
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/nfs-servers.yml
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/noc.yml
 - import_playbook: /srv/web/infra/ansible/playbooks/groups/os-control.yml

playbooks/include/proxies-reverseproxy.yml

@@ -440,12 +440,25 @@
     keephost: true
     tags: transtats
 
+  - role: httpd/reverseproxy
+    website: "mirrormanager{{ env_suffix }}.fedoraproject.org"
+    destname: mirrormanager
+    balancer_name: apps-ocp
+    balancer_members: "{{ (env == 'staging')|ternary(ocp_nodes_stg, ocp_nodes) }}"
+    targettype: openshift
+    keephost: true
+    ocp4: true
+    tags:
+    - mirrormanager
+
   - role: httpd/reverseproxy
     website: admin.fedoraproject.org
     destname: mirrormanager
     remotepath: /mirrormanager
     localpath: /mirrormanager
     proxyurl: "{{ varnish_url }}"
+    tags:
+    - mirrormanager
 
   - role: httpd/reverseproxy
     website: mirrors.fedoraproject.org

playbooks/include/proxies-websites.yml

@@ -168,6 +168,13 @@
     tags: mirrors.stg.centos.org
     when: env == "staging"
 
+  - role: httpd/website
+    site_name: "mirrormanager{{ env_suffix }}.fedoraproject.org"
+    sslonly: true
+    cert_name: "{{wildcard_cert_name}}"
+    tags:
+      - mirrormanager
+
   - role: httpd/website
     site_name: src.fedoraproject.org
     server_aliases: [src.stg.fedoraproject.org]

playbooks/openshift-apps/mirrormanager.yml

@@ -94,6 +94,15 @@
       serviceport: web
       annotations:
         haproxy.router.openshift.io/timeout: 5m
+    - role: openshift/route
+      app: mirrormanager
+      routename: frontend
+      host: "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org"
+      servicename: frontend
+      serviceport: web
+      annotations:
+        haproxy.router.openshift.io/timeout: 5m
+      when: mirrormanager_hostname != "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org"
 
     # SSH key to get the access logs
     - role: openshift/secret-file

roles/mirrormanager/backend/files/backend.cron

@@ -1,31 +1,33 @@
 MAILTO=root
 
-# Refresh the mirrorlist cache every 15 minutes
+## Replaced by mirrormanager in OpenShift
-# This takes about 1 minute
+#
-0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
+# # Refresh the mirrorlist cache every 15 minutes
-
+# # This takes about 1 minute
-# update master directory
+# 0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
-# logs sent to /var/log/mirrormanager/umdl.log by default
+# 
-# check if category Fedora EPEL needs updating every 30 minutes
+# # update master directory
-0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log
+# # logs sent to /var/log/mirrormanager/umdl.log by default
-# check if category Fedora Linux needs updating every 30 minutes
+# # check if category Fedora EPEL needs updating every 30 minutes
-10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
+# 0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log
-# check if category Fedora Secondary Arches needs updating every 30 minutes
+# # check if category Fedora Linux needs updating every 30 minutes
-20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
+# 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
-# check if category Fedora Codecs needs updating once a day
+# # check if category Fedora Secondary Arches needs updating every 30 minutes
-15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
+# 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
-# the remaining categories are updated every two hours
+# # check if category Fedora Codecs needs updating once a day
-15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
+# 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
-45 */2  * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
+# # the remaining categories are updated every two hours
-
+# 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
-# check if category CentOS needs updating every 15 minutes
+# 45 */2  * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
-57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
+# 
-
+# # check if category CentOS needs updating every 15 minutes
-# Sync netblocks list once a day
+# 57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
-30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
+# 
-# Run the script for Internet2 netblocks very late to
+# # Sync netblocks list once a day
-# make sure the files are all there. The script is
+# 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
-# looking in date dependent directories and depending
+# # Run the script for Internet2 netblocks very late to
-# on the timezone of the remote server the files appear late.
+# # make sure the files are all there. The script is
-0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt
+# # looking in date dependent directories and depending
-30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks
+# # on the timezone of the remote server the files appear late.
+# 0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt
+# 30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks

roles/mirrormanager/backend/tasks/main.yml

@@ -67,21 +67,25 @@
   cron: name="handle-propagation" minute="45" hour="*/2" user="mirrormanager"
         job="/usr/local/bin/handle_propagation.sh"
         cron_file=handle-propagation
+        disabled=true
 
 - name: create worldmap cronjob
   cron: name="create-worldmap" minute="50" hour="20" user="mirrormanager"
         job="/usr/local/bin/create_maps.sh"
         cron_file=create-worldmap
+        disabled=true
 
 - name: create mirrorlist statistics cronjob
   cron: name="create-statistics" minute="4" hour="*/2" user="mirrormanager"
         job="/usr/local/bin/create_statistics.sh"
         cron_file=create-statistics
+        disabled=true
 
 - name: create yesterdays mirrorlist statistics cronjob
   cron: name="yesterdays-statistics" minute="55" hour="0" user="mirrormanager"
         job="/usr/local/bin/create_statistics.sh yesterday"
         cron_file=yesterdays-statistics
+        disabled=true
 
 - name: setup /var/lib/mirrormanager/.ssh directory
   copy: >

roles/mirrormanager/crawler/files/crawler.cron

@@ -1,38 +1,40 @@
-# run the crawler for each MirrorManager category
+## Replaced by mirrormanager in OpenShift
-# logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
 #
-# [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl
+# # run the crawler for each MirrorManager category
-# later on the second crawler to reduce the number of parallel accesses to
+# # logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
-# the database
+# #
+# # [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl
+# # later on the second crawler to reduce the number of parallel accesses to
+# # the database
+# #
+# # To make sure only one cron started crawler is running the previous running
+# # (cron) crawlers are being signaled to shut down.  The crawler can try to
+# # gracefully shutdown if it gets the signal SIGALRM(14).  After the signal we
+# # wait for 5 minutes to give the crawler a chance to shutdown. After that the
+# # crawler is killed.  To make sure we only end the cron started crawler we look
+# # for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25".
 # 
-# To make sure only one cron started crawler is running the previous running
+# # The number of threads is based on the possible number of existing mirrors. More
-# (cron) crawlers are being signaled to shut down.  The crawler can try to
+# # threads for categories with more mirrors.
-# gracefully shutdown if it gets the signal SIGALRM(14).  After the signal we
+# 
-# wait for 5 minutes to give the crawler a chance to shutdown. After that the
+# # The goal is to distribute the crawling of all categories over the whole day.
-# crawler is killed.  To make sure we only end the cron started crawler we look
+# 
-# for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25".
+# # The timeout is 4 hours, but for each category.
-
+# 
-# The number of threads is based on the possible number of existing mirrors. More
+# # Category: 'Fedora Linux'; twice a day, 20 threads
-# threads for categories with more mirrors.
+# 0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-
+# 
-# The goal is to distribute the crawling of all categories over the whole day.
+# # Category: 'Fedora Secondary Arches'; twice a day, 10 threads
-
+# 0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-# The timeout is 4 hours, but for each category.
+# 
-
+# # Category: 'Fedora EPEL'; four times a day, 20 threads
-# Category: 'Fedora Linux'; twice a day, 20 threads
+# 45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
+# 
-
+# # Category: 'Fedora Archive'; once a day, 10 threads
-# Category: 'Fedora Secondary Arches'; twice a day, 10 threads
+# 0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
+# 
-
+# # Category: 'Fedora Other'; once a day, 10 threads
-# Category: 'Fedora EPEL'; four times a day, 20 threads
+# 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
+# 
-
+# # Category: 'CentOS'; twice a day, 20 threads
-# Category: 'Fedora Archive'; once a day, 10 threads
+# 0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-
-# Category: 'Fedora Other'; once a day, 10 threads
-0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
-
-# Category: 'CentOS'; twice a day, 20 threads
-0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1

roles/mirrormanager/crawler/tasks/main.yml

@@ -51,4 +51,5 @@
   cron: name="propagation-check" minute="27" hour="*/2" user="mirrormanager"
         job="/usr/local/bin/check_propagation.sh"
         cron_file=propagation-check
+        disabled=true
   when: inventory_hostname.startswith('mm-crawler01.iad2')

roles/mirrormanager/frontend2/templates/frontend.cron

@@ -1,4 +1,6 @@
 MAILTO=root
 
-# sync crawler logs every hour
+## Replaced by mirrormanager in OpenShift
-55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh
+#
+# # sync crawler logs every hour
+# 55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh

roles/mirrormanager/frontend2/templates/mirrormanager.conf

@@ -1,35 +1,39 @@
-# Apache configuration file for mirrormanager2
+## Replaced by mirrormanager in OpenShift
 
-Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/
+Redirect /mirrormanager https://mirrormanager{{ env_suffix }}.fedoraproject.org
-Alias /mirrormanager/crawler /var/log/mirrormanager/crawler
-Alias /mirrormanager/data /var/www/mirrormanager-statistics/data
-Alias /mirrormanager/map /var/www/mirrormanager-statistics/map
 
-WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4
+# # Apache configuration file for mirrormanager2
-WSGISocketPrefix run/wsgi
+# 
-WSGIRestrictStdout On
+# Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/
-WSGIRestrictSignal Off
+# Alias /mirrormanager/crawler /var/log/mirrormanager/crawler
-WSGIPythonOptimize 1
+# Alias /mirrormanager/data /var/www/mirrormanager-statistics/data
-
+# Alias /mirrormanager/map /var/www/mirrormanager-statistics/map
-WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi
+# 
-
+# WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4
-<Location />
+# WSGISocketPrefix run/wsgi
-    WSGIProcessGroup mirrormanager
+# WSGIRestrictStdout On
-    <IfModule mod_authz_core.c>
+# WSGIRestrictSignal Off
-        # Apache 2.4
+# WSGIPythonOptimize 1
-        Require all granted
+# 
-    </IfModule>
+# WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi
-    <IfModule !mod_authz_core.c>
+# 
-         # Apache 2.2
+# <Location />
-        Order deny,allow
+#     WSGIProcessGroup mirrormanager
-        Allow from all
+#     <IfModule mod_authz_core.c>
-    </IfModule>
+#         # Apache 2.4
-</Location>
+#         Require all granted
-
+#     </IfModule>
-<Location /mirrormanager/xmlrpc>
+#     <IfModule !mod_authz_core.c>
-{% if mm2_checkin %}
+#          # Apache 2.2
-	Require all granted
+#         Order deny,allow
-{% else %}
+#         Allow from all
-	Require all denied
+#     </IfModule>
-{% endif %}
+# </Location>
-</Location>
+# 
+# <Location /mirrormanager/xmlrpc>
+# {% if mm2_checkin %}
+# 	Require all granted
+# {% else %}
+# 	Require all denied
+# {% endif %}
+# </Location>

vars/apps/mirrormanager.yml

@@ -1,5 +1,5 @@
 ---
-mirrormanager_hostname: "mirrors.apps.ocp{{ env_suffix }}.fedoraproject.org"
+mirrormanager_hostname: "mirrormanager{{ env_suffix }}.fedoraproject.org"
 mirrormanager_db_host: "db01{{ env_suffix }}"
 mirrormanager_oidc_client_id: "mirrormanager"
 # Move that to an env var to target a specific deployment