Switch mirrormanager to the Openshift version

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard 2024-05-29 09:14:09 +02:00
parent 22831e10e6
commit 02b5245e28
No known key found for this signature in database
GPG key ID: 31584CFEB9BF64AD
11 changed files with 146 additions and 103 deletions

View file

@ -42,7 +42,6 @@
- import_playbook: /srv/web/infra/ansible/playbooks/groups/maintainer-test.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/mariadb-server.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/memcached.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/mirrormanager.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/nfs-servers.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/noc.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/os-control.yml

View file

@ -440,12 +440,25 @@
keephost: true
tags: transtats
- role: httpd/reverseproxy
website: "mirrormanager{{ env_suffix }}.fedoraproject.org"
destname: mirrormanager
balancer_name: apps-ocp
balancer_members: "{{ (env == 'staging')|ternary(ocp_nodes_stg, ocp_nodes) }}"
targettype: openshift
keephost: true
ocp4: true
tags:
- mirrormanager
- role: httpd/reverseproxy
website: admin.fedoraproject.org
destname: mirrormanager
remotepath: /mirrormanager
localpath: /mirrormanager
proxyurl: "{{ varnish_url }}"
tags:
- mirrormanager
- role: httpd/reverseproxy
website: mirrors.fedoraproject.org

View file

@ -168,6 +168,13 @@
tags: mirrors.stg.centos.org
when: env == "staging"
- role: httpd/website
site_name: "mirrormanager{{ env_suffix }}.fedoraproject.org"
sslonly: true
cert_name: "{{wildcard_cert_name}}"
tags:
- mirrormanager
- role: httpd/website
site_name: src.fedoraproject.org
server_aliases: [src.stg.fedoraproject.org]

View file

@ -94,6 +94,15 @@
serviceport: web
annotations:
haproxy.router.openshift.io/timeout: 5m
- role: openshift/route
app: mirrormanager
routename: frontend
host: "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org"
servicename: frontend
serviceport: web
annotations:
haproxy.router.openshift.io/timeout: 5m
when: mirrormanager_hostname != "mirrormanager.apps.ocp{{ env_suffix }}.fedoraproject.org"
# SSH key to get the access logs
- role: openshift/secret-file

View file

@ -1,31 +1,33 @@
MAILTO=root
# Refresh the mirrorlist cache every 15 minutes
# This takes about 1 minute
0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
# update master directory
# logs sent to /var/log/mirrormanager/umdl.log by default
# check if category Fedora EPEL needs updating every 30 minutes
0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log
# check if category Fedora Linux needs updating every 30 minutes
10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
# check if category Fedora Secondary Arches needs updating every 30 minutes
20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
# check if category Fedora Codecs needs updating once a day
15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
# the remaining categories are updated every two hours
15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
# check if category CentOS needs updating every 15 minutes
57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
# Sync netblocks list once a day
30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
# Run the script for Internet2 netblocks very late to
# make sure the files are all there. The script is
# looking in date dependent directories and depending
# on the timezone of the remote server the files appear late.
0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt
30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks
## Replaced by mirrormanager in OpenShift
#
# # Refresh the mirrorlist cache every 15 minutes
# # This takes about 1 minute
# 0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
#
# # update master directory
# # logs sent to /var/log/mirrormanager/umdl.log by default
# # check if category Fedora EPEL needs updating every 30 minutes
# 0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log
# # check if category Fedora Linux needs updating every 30 minutes
# 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
# # check if category Fedora Secondary Arches needs updating every 30 minutes
# 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
# # check if category Fedora Codecs needs updating once a day
# 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
# # the remaining categories are updated every two hours
# 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
# 45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
#
# # check if category CentOS needs updating every 15 minutes
# 57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
#
# # Sync netblocks list once a day
# 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
# # Run the script for Internet2 netblocks very late to
# # make sure the files are all there. The script is
# # looking in date dependent directories and depending
# # on the timezone of the remote server the files appear late.
# 0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt
# 30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks

View file

@ -67,21 +67,25 @@
cron: name="handle-propagation" minute="45" hour="*/2" user="mirrormanager"
job="/usr/local/bin/handle_propagation.sh"
cron_file=handle-propagation
disabled=true
- name: create worldmap cronjob
cron: name="create-worldmap" minute="50" hour="20" user="mirrormanager"
job="/usr/local/bin/create_maps.sh"
cron_file=create-worldmap
disabled=true
- name: create mirrorlist statistics cronjob
cron: name="create-statistics" minute="4" hour="*/2" user="mirrormanager"
job="/usr/local/bin/create_statistics.sh"
cron_file=create-statistics
disabled=true
- name: create yesterdays mirrorlist statistics cronjob
cron: name="yesterdays-statistics" minute="55" hour="0" user="mirrormanager"
job="/usr/local/bin/create_statistics.sh yesterday"
cron_file=yesterdays-statistics
disabled=true
- name: setup /var/lib/mirrormanager/.ssh directory
copy: >

View file

@ -1,38 +1,40 @@
# run the crawler for each MirrorManager category
# logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
## Replaced by mirrormanager in OpenShift
#
# [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl
# later on the second crawler to reduce the number of parallel accesses to
# the database
#
# To make sure only one cron started crawler is running the previous running
# (cron) crawlers are being signaled to shut down. The crawler can try to
# gracefully shutdown if it gets the signal SIGALRM(14). After the signal we
# wait for 5 minutes to give the crawler a chance to shutdown. After that the
# crawler is killed. To make sure we only end the cron started crawler we look
# for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25".
# The number of threads is based on the possible number of existing mirrors. More
# threads for categories with more mirrors.
# The goal is to distribute the crawling of all categories over the whole day.
# The timeout is 4 hours, but for each category.
# Category: 'Fedora Linux'; twice a day, 20 threads
0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# Category: 'Fedora Secondary Arches'; twice a day, 10 threads
0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# Category: 'Fedora EPEL'; four times a day, 20 threads
45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# Category: 'Fedora Archive'; once a day, 10 threads
0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# Category: 'Fedora Other'; once a day, 10 threads
0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# Category: 'CentOS'; twice a day, 20 threads
0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
# # run the crawler for each MirrorManager category
# # logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
# #
# # [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl
# # later on the second crawler to reduce the number of parallel accesses to
# # the database
# #
# # To make sure only one cron started crawler is running the previous running
# # (cron) crawlers are being signaled to shut down. The crawler can try to
# # gracefully shutdown if it gets the signal SIGALRM(14). After the signal we
# # wait for 5 minutes to give the crawler a chance to shutdown. After that the
# # crawler is killed. To make sure we only end the cron started crawler we look
# # for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25".
#
# # The number of threads is based on the possible number of existing mirrors. More
# # threads for categories with more mirrors.
#
# # The goal is to distribute the crawling of all categories over the whole day.
#
# # The timeout is 4 hours, but for each category.
#
# # Category: 'Fedora Linux'; twice a day, 20 threads
# 0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
#
# # Category: 'Fedora Secondary Arches'; twice a day, 10 threads
# 0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
#
# # Category: 'Fedora EPEL'; four times a day, 20 threads
# 45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
#
# # Category: 'Fedora Archive'; once a day, 10 threads
# 0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
#
# # Category: 'Fedora Other'; once a day, 10 threads
# 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
#
# # Category: 'CentOS'; twice a day, 20 threads
# 0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1

View file

@ -51,4 +51,5 @@
cron: name="propagation-check" minute="27" hour="*/2" user="mirrormanager"
job="/usr/local/bin/check_propagation.sh"
cron_file=propagation-check
disabled=true
when: inventory_hostname.startswith('mm-crawler01.iad2')

View file

@ -1,4 +1,6 @@
MAILTO=root
# sync crawler logs every hour
55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh
## Replaced by mirrormanager in OpenShift
#
# # sync crawler logs every hour
# 55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh

View file

@ -1,35 +1,39 @@
# Apache configuration file for mirrormanager2
## Replaced by mirrormanager in OpenShift
Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/
Alias /mirrormanager/crawler /var/log/mirrormanager/crawler
Alias /mirrormanager/data /var/www/mirrormanager-statistics/data
Alias /mirrormanager/map /var/www/mirrormanager-statistics/map
Redirect /mirrormanager https://mirrormanager{{ env_suffix }}.fedoraproject.org
WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4
WSGISocketPrefix run/wsgi
WSGIRestrictStdout On
WSGIRestrictSignal Off
WSGIPythonOptimize 1
WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi
<Location />
WSGIProcessGroup mirrormanager
<IfModule mod_authz_core.c>
# Apache 2.4
Require all granted
</IfModule>
<IfModule !mod_authz_core.c>
# Apache 2.2
Order deny,allow
Allow from all
</IfModule>
</Location>
<Location /mirrormanager/xmlrpc>
{% if mm2_checkin %}
Require all granted
{% else %}
Require all denied
{% endif %}
</Location>
# # Apache configuration file for mirrormanager2
#
# Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/
# Alias /mirrormanager/crawler /var/log/mirrormanager/crawler
# Alias /mirrormanager/data /var/www/mirrormanager-statistics/data
# Alias /mirrormanager/map /var/www/mirrormanager-statistics/map
#
# WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4
# WSGISocketPrefix run/wsgi
# WSGIRestrictStdout On
# WSGIRestrictSignal Off
# WSGIPythonOptimize 1
#
# WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi
#
# <Location />
# WSGIProcessGroup mirrormanager
# <IfModule mod_authz_core.c>
# # Apache 2.4
# Require all granted
# </IfModule>
# <IfModule !mod_authz_core.c>
# # Apache 2.2
# Order deny,allow
# Allow from all
# </IfModule>
# </Location>
#
# <Location /mirrormanager/xmlrpc>
# {% if mm2_checkin %}
# Require all granted
# {% else %}
# Require all denied
# {% endif %}
# </Location>

View file

@ -1,5 +1,5 @@
---
mirrormanager_hostname: "mirrors.apps.ocp{{ env_suffix }}.fedoraproject.org"
mirrormanager_hostname: "mirrormanager{{ env_suffix }}.fedoraproject.org"
mirrormanager_db_host: "db01{{ env_suffix }}"
mirrormanager_oidc_client_id: "mirrormanager"
# Move that to an env var to target a specific deployment