diff --git a/main.yml b/main.yml index ca1ba801a4..f6e678a406 100644 --- a/main.yml +++ b/main.yml @@ -42,7 +42,6 @@ - import_playbook: /srv/web/infra/ansible/playbooks/groups/maintainer-test.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/mariadb-server.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/memcached.yml -- import_playbook: /srv/web/infra/ansible/playbooks/groups/mirrormanager.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/nfs-servers.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/noc.yml - import_playbook: /srv/web/infra/ansible/playbooks/groups/os-control.yml diff --git a/playbooks/include/proxies-reverseproxy.yml b/playbooks/include/proxies-reverseproxy.yml index 363c4c293c..142cdf3317 100644 --- a/playbooks/include/proxies-reverseproxy.yml +++ b/playbooks/include/proxies-reverseproxy.yml @@ -440,12 +440,25 @@ keephost: true tags: transtats + - role: httpd/reverseproxy + website: "mirrormanager{{ env_suffix }}.fedoraproject.org" + destname: mirrormanager + balancer_name: apps-ocp + balancer_members: "{{ (env == 'staging')|ternary(ocp_nodes_stg, ocp_nodes) }}" + targettype: openshift + keephost: true + ocp4: true + tags: + - mirrormanager + - role: httpd/reverseproxy website: admin.fedoraproject.org destname: mirrormanager remotepath: /mirrormanager localpath: /mirrormanager proxyurl: "{{ varnish_url }}" + tags: + - mirrormanager - role: httpd/reverseproxy website: mirrors.fedoraproject.org diff --git a/playbooks/include/proxies-websites.yml b/playbooks/include/proxies-websites.yml index 35f5109ef8..64f1139594 100644 --- a/playbooks/include/proxies-websites.yml +++ b/playbooks/include/proxies-websites.yml @@ -168,6 +168,13 @@ tags: mirrors.stg.centos.org when: env == "staging" + - role: httpd/website + site_name: "mirrormanager{{ env_suffix }}.fedoraproject.org" + sslonly: true + cert_name: "{{wildcard_cert_name}}" + tags: + - mirrormanager + - role: httpd/website site_name: src.fedoraproject.org server_aliases: [src.stg.fedoraproject.org] diff --git a/playbooks/openshift-apps/mirrormanager.yml b/playbooks/openshift-apps/mirrormanager.yml index 1f9a96ed4f..01e7b52a62 100644 --- a/playbooks/openshift-apps/mirrormanager.yml +++ b/playbooks/openshift-apps/mirrormanager.yml @@ -94,6 +94,15 @@ serviceport: web annotations: haproxy.router.openshift.io/timeout: 5m + - role: openshift/route + app: mirrormanager + routename: frontend + host: "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org" + servicename: frontend + serviceport: web + annotations: + haproxy.router.openshift.io/timeout: 5m + when: mirrormanager_hostname != "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org" # SSH key to get the access logs - role: openshift/secret-file diff --git a/roles/mirrormanager/backend/files/backend.cron b/roles/mirrormanager/backend/files/backend.cron index fd3a15bd54..90eb58d146 100644 --- a/roles/mirrormanager/backend/files/backend.cron +++ b/roles/mirrormanager/backend/files/backend.cron @@ -1,31 +1,33 @@ MAILTO=root -# Refresh the mirrorlist cache every 15 minutes -# This takes about 1 minute -0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh - -# update master directory -# logs sent to /var/log/mirrormanager/umdl.log by default -# check if category Fedora EPEL needs updating every 30 minutes -0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log -# check if category Fedora Linux needs updating every 30 minutes -10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log -# check if category Fedora Secondary Arches needs updating every 30 minutes -20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log -# check if category Fedora Codecs needs updating once a day -15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log -# the remaining categories are updated every two hours -15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log -45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log - -# check if category CentOS needs updating every 15 minutes -57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log - -# Sync netblocks list once a day -30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt -# Run the script for Internet2 netblocks very late to -# make sure the files are all there. The script is -# looking in date dependent directories and depending -# on the timezone of the remote server the files appear late. -0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt -30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks +## Replaced by mirrormanager in OpenShift +# +# # Refresh the mirrorlist cache every 15 minutes +# # This takes about 1 minute +# 0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh +# +# # update master directory +# # logs sent to /var/log/mirrormanager/umdl.log by default +# # check if category Fedora EPEL needs updating every 30 minutes +# 0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log +# # check if category Fedora Linux needs updating every 30 minutes +# 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log +# # check if category Fedora Secondary Arches needs updating every 30 minutes +# 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log +# # check if category Fedora Codecs needs updating once a day +# 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log +# # the remaining categories are updated every two hours +# 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log +# 45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log +# +# # check if category CentOS needs updating every 15 minutes +# 57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log +# +# # Sync netblocks list once a day +# 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt +# # Run the script for Internet2 netblocks very late to +# # make sure the files are all there. The script is +# # looking in date dependent directories and depending +# # on the timezone of the remote server the files appear late. +# 0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt +# 30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks diff --git a/roles/mirrormanager/backend/tasks/main.yml b/roles/mirrormanager/backend/tasks/main.yml index 2926922d5e..bbc64657a5 100644 --- a/roles/mirrormanager/backend/tasks/main.yml +++ b/roles/mirrormanager/backend/tasks/main.yml @@ -67,21 +67,25 @@ cron: name="handle-propagation" minute="45" hour="*/2" user="mirrormanager" job="/usr/local/bin/handle_propagation.sh" cron_file=handle-propagation + disabled=true - name: create worldmap cronjob cron: name="create-worldmap" minute="50" hour="20" user="mirrormanager" job="/usr/local/bin/create_maps.sh" cron_file=create-worldmap + disabled=true - name: create mirrorlist statistics cronjob cron: name="create-statistics" minute="4" hour="*/2" user="mirrormanager" job="/usr/local/bin/create_statistics.sh" cron_file=create-statistics + disabled=true - name: create yesterdays mirrorlist statistics cronjob cron: name="yesterdays-statistics" minute="55" hour="0" user="mirrormanager" job="/usr/local/bin/create_statistics.sh yesterday" cron_file=yesterdays-statistics + disabled=true - name: setup /var/lib/mirrormanager/.ssh directory copy: > diff --git a/roles/mirrormanager/crawler/files/crawler.cron b/roles/mirrormanager/crawler/files/crawler.cron index b2fb775abf..16541f1a0c 100644 --- a/roles/mirrormanager/crawler/files/crawler.cron +++ b/roles/mirrormanager/crawler/files/crawler.cron @@ -1,38 +1,40 @@ -# run the crawler for each MirrorManager category -# logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default +## Replaced by mirrormanager in OpenShift # -# [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl -# later on the second crawler to reduce the number of parallel accesses to -# the database -# -# To make sure only one cron started crawler is running the previous running -# (cron) crawlers are being signaled to shut down. The crawler can try to -# gracefully shutdown if it gets the signal SIGALRM(14). After the signal we -# wait for 5 minutes to give the crawler a chance to shutdown. After that the -# crawler is killed. To make sure we only end the cron started crawler we look -# for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25". - -# The number of threads is based on the possible number of existing mirrors. More -# threads for categories with more mirrors. - -# The goal is to distribute the crawling of all categories over the whole day. - -# The timeout is 4 hours, but for each category. - -# Category: 'Fedora Linux'; twice a day, 20 threads -0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 - -# Category: 'Fedora Secondary Arches'; twice a day, 10 threads -0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 - -# Category: 'Fedora EPEL'; four times a day, 20 threads -45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 - -# Category: 'Fedora Archive'; once a day, 10 threads -0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 - -# Category: 'Fedora Other'; once a day, 10 threads -0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 - -# Category: 'CentOS'; twice a day, 20 threads -0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# # run the crawler for each MirrorManager category +# # logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default +# # +# # [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl +# # later on the second crawler to reduce the number of parallel accesses to +# # the database +# # +# # To make sure only one cron started crawler is running the previous running +# # (cron) crawlers are being signaled to shut down. The crawler can try to +# # gracefully shutdown if it gets the signal SIGALRM(14). After the signal we +# # wait for 5 minutes to give the crawler a chance to shutdown. After that the +# # crawler is killed. To make sure we only end the cron started crawler we look +# # for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25". +# +# # The number of threads is based on the possible number of existing mirrors. More +# # threads for categories with more mirrors. +# +# # The goal is to distribute the crawling of all categories over the whole day. +# +# # The timeout is 4 hours, but for each category. +# +# # Category: 'Fedora Linux'; twice a day, 20 threads +# 0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# +# # Category: 'Fedora Secondary Arches'; twice a day, 10 threads +# 0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# +# # Category: 'Fedora EPEL'; four times a day, 20 threads +# 45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# +# # Category: 'Fedora Archive'; once a day, 10 threads +# 0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# +# # Category: 'Fedora Other'; once a day, 10 threads +# 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +# +# # Category: 'CentOS'; twice a day, 20 threads +# 0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 diff --git a/roles/mirrormanager/crawler/tasks/main.yml b/roles/mirrormanager/crawler/tasks/main.yml index 867e9b4a77..42f440257a 100644 --- a/roles/mirrormanager/crawler/tasks/main.yml +++ b/roles/mirrormanager/crawler/tasks/main.yml @@ -51,4 +51,5 @@ cron: name="propagation-check" minute="27" hour="*/2" user="mirrormanager" job="/usr/local/bin/check_propagation.sh" cron_file=propagation-check + disabled=true when: inventory_hostname.startswith('mm-crawler01.iad2') diff --git a/roles/mirrormanager/frontend2/templates/frontend.cron b/roles/mirrormanager/frontend2/templates/frontend.cron index 6725774b2d..1f9f3fa296 100644 --- a/roles/mirrormanager/frontend2/templates/frontend.cron +++ b/roles/mirrormanager/frontend2/templates/frontend.cron @@ -1,4 +1,6 @@ MAILTO=root -# sync crawler logs every hour -55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh +## Replaced by mirrormanager in OpenShift +# +# # sync crawler logs every hour +# 55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh diff --git a/roles/mirrormanager/frontend2/templates/mirrormanager.conf b/roles/mirrormanager/frontend2/templates/mirrormanager.conf index de4b3a41d2..cde585fa74 100644 --- a/roles/mirrormanager/frontend2/templates/mirrormanager.conf +++ b/roles/mirrormanager/frontend2/templates/mirrormanager.conf @@ -1,35 +1,39 @@ -# Apache configuration file for mirrormanager2 +## Replaced by mirrormanager in OpenShift -Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/ -Alias /mirrormanager/crawler /var/log/mirrormanager/crawler -Alias /mirrormanager/data /var/www/mirrormanager-statistics/data -Alias /mirrormanager/map /var/www/mirrormanager-statistics/map +Redirect /mirrormanager https://mirrormanager{{ env_suffix }}.fedoraproject.org -WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4 -WSGISocketPrefix run/wsgi -WSGIRestrictStdout On -WSGIRestrictSignal Off -WSGIPythonOptimize 1 - -WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi - - - WSGIProcessGroup mirrormanager - - # Apache 2.4 - Require all granted - - - # Apache 2.2 - Order deny,allow - Allow from all - - - - -{% if mm2_checkin %} - Require all granted -{% else %} - Require all denied -{% endif %} - +# # Apache configuration file for mirrormanager2 +# +# Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/ +# Alias /mirrormanager/crawler /var/log/mirrormanager/crawler +# Alias /mirrormanager/data /var/www/mirrormanager-statistics/data +# Alias /mirrormanager/map /var/www/mirrormanager-statistics/map +# +# WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4 +# WSGISocketPrefix run/wsgi +# WSGIRestrictStdout On +# WSGIRestrictSignal Off +# WSGIPythonOptimize 1 +# +# WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi +# +# +# WSGIProcessGroup mirrormanager +# +# # Apache 2.4 +# Require all granted +# +# +# # Apache 2.2 +# Order deny,allow +# Allow from all +# +# +# +# +# {% if mm2_checkin %} +# Require all granted +# {% else %} +# Require all denied +# {% endif %} +# diff --git a/vars/apps/mirrormanager.yml b/vars/apps/mirrormanager.yml index 175c28213d..d804a5382c 100644 --- a/vars/apps/mirrormanager.yml +++ b/vars/apps/mirrormanager.yml @@ -1,5 +1,5 @@ --- -mirrormanager_hostname: "mirrors.apps.ocp{{ env_suffix }}.fedoraproject.org" +mirrormanager_hostname: "mirrormanager{{ env_suffix }}.fedoraproject.org" mirrormanager_db_host: "db01{{ env_suffix }}" mirrormanager_oidc_client_id: "mirrormanager" # Move that to an env var to target a specific deployment