diff --git a/roles/haproxy/templates/haproxy.cfg b/roles/haproxy/templates/haproxy.cfg index 50b2e0144c..8a68ef731b 100644 --- a/roles/haproxy/templates/haproxy.cfg +++ b/roles/haproxy/templates/haproxy.cfg @@ -58,9 +58,6 @@ backend mirror-lists-backend timeout connect 30s server mirrorlist-local1 127.0.0.1:18081 check inter 1s rise 2 fall 3 weight 100 server mirrorlist-local2 127.0.0.1:18082 check inter 1s rise 2 fall 3 weight 100 -{% if datacenter == 'iad2' %} - server mirrorlist-local3 127.0.0.1:18083 check inter 1s rise 2 fall 3 weight 100 -{% endif %} option httpchk GET /metalink?repo=epel-7&arch=x86_64 option allbackups diff --git a/roles/mirrormanager/backend/templates/create_statistics.sh b/roles/mirrormanager/backend/templates/create_statistics.sh index b801a492a5..0bbc37fd42 100644 --- a/roles/mirrormanager/backend/templates/create_statistics.sh +++ b/roles/mirrormanager/backend/templates/create_statistics.sh @@ -6,7 +6,6 @@ FRONTENDS="{% for host in groups['mm_frontend'] %} {{ host }} {% endfor %}" INPUT="/var/log/mirrormanager/mirrorlist.log" CONTAINER1="/var/log/mirrormanager/mirrorlist1.service.log" CONTAINER2="/var/log/mirrormanager/mirrorlist2.service.log" -CONTAINER3="/var/log/mirrormanager/mirrorlist3.service.log" if [ "$1" == "yesterday" ]; then STATISTICS="/usr/bin/mirrorlist_statistics -o 1" @@ -24,11 +23,9 @@ trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT for s in ${MIRRORLIST_PROXIES}; do ssh $s "( cat $CONTAINER1 | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz ssh $s "( cat $CONTAINER2 | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz - ssh $s "( cat $CONTAINER3 | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz 2>/dev/null if [ "$1" == "yesterday" ]; then ssh $s "( xzcat $CONTAINER1-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz ssh $s "( xzcat $CONTAINER2-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz - ssh $s "( xzcat $CONTAINER3-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz 2>/dev/null fi done diff --git a/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml b/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml deleted file mode 100644 index aa608f04c2..0000000000 --- a/roles/mirrormanager/mirrorlist_proxy/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -mirrorlist_container_image: "quay.io/adrianreber/mirrorlist-server:2.0.0" -mirrorlist_container_image_stg: "quay.io/adrianreber/mirrorlist-server:2.0.0" diff --git a/roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 b/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-processes similarity index 61% rename from roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 rename to roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-processes index 6435304464..b320978b52 100644 --- a/roles/mirrormanager/mirrorlist_proxy/templates/restart-mirrorlist-containers.j2 +++ b/roles/mirrormanager/mirrorlist_proxy/files/restart-mirrorlist-processes @@ -1,18 +1,12 @@ #!/bin/bash # -# This job handles hourly restarting mirrorlist container(s) and making sure all is well. -# -# Starting with Fedora 31 this no longer uses containers. +# This job handles restarting the mirrorlist process and making sure all is well. # -## -## Docker ports for mirrors +## ports for mirrorlist processes mirrorlist1="http://localhost:18081/metalink?repo=rawhide&arch=x86_64" mirrorlist2="http://localhost:18082/metalink?repo=rawhide&arch=x86_64" -{% if datacenter == 'iad2' %} -mirrorlist3="http://localhost:18083/metalink?repo=rawhide&arch=x86_64" -{% endif %} TIME_DRAIN=30 TIME_RESTART=5 @@ -27,15 +21,6 @@ then cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist2/ fi -{% if datacenter == 'iad2' %} -# We have more mirror containers here -if [ ! -f /srv/mirrorlist/data/mirrorlist3/global_netblocks.txt ]; -then - cp /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/ -fi -{% endif %} - - ## Check that protbuf cache is newer than old protobuf cache if [ /srv/mirrorlist/data/mirrorlist1/mirrorlist_cache.proto -nt /srv/mirrorlist/data/mirrorlist2/mirrorlist_cache.proto ]; then # new proto @@ -58,16 +43,6 @@ if [ $? != 0 ]; then exit 1 fi -{% if datacenter == 'iad2' %} -# check mirrorlist3 (old protbuf cache and see that it's processing ok) -curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 | grep "sha512" >/dev/null -if [ $? != 0 ]; then - echo "ERROR: mirrorlist3 not processing correctly" - exit 1 -fi -{% endif %} - - # Drain mirrorlist1. This is safe since we assured that local2 is serving echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null echo "set server mirror-lists-backend/mirrorlist-local1 state drain" | nc -U /var/run/haproxy-admin >& /dev/null @@ -80,11 +55,6 @@ systemctl stop mirrorlist1 sleep 1 systemctl start mirrorlist1 if [[ ${?} -ne 0 ]]; then -{% if ansible_distribution_major_version|int < 31 %} - # We have seen this to fail because podman was not able to completely - # remove the container for some reason. First try this: - sudo -u mirrormanager /usr/bin/podman rm mirrorlist1.service --storage --force -{% endif %} systemctl start mirrorlist1 if [[ ${?} -ne 0 ]]; then echo "Unable to start mirrorlist1" @@ -130,32 +100,3 @@ if [ $? != 0 ]; then fi echo "enable server mirror-lists-backend/mirrorlist-local2" | nc -U /var/run/haproxy-admin >& /dev/null - -{% if datacenter == 'iad2' %} -# copy new protbuf cache to mirrorlist3 -cp -a /srv/mirrorlist/data/mirrorlist1/* /srv/mirrorlist/data/mirrorlist3/ - -# Drain mirrorlist3 -echo "set server mirror-lists-backend/mirrorlist-local3 state drain" | nc -U /var/run/haproxy-admin >& /dev/null -sleep ${TIME_DRAIN} -echo "disable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null -sleep ${TIME_DISABLE} - -# restart mirrorlist3 -systemctl stop mirrorlist3 -sleep 1 -systemctl start mirrorlist3 -if [[ ${?} -ne 0 ]]; then - echo "Unable to start mirrorlist3" - exit 1 -fi - -sleep ${TIME_RESTART} -curl -q -H mirrors.fedoraproject.org ${mirrorlist3} -o/dev/null -s -f --retry 50 --retry-delay 10 --retry-connrefused --retry-max-time 180 -if [ $? != 0 ]; then - echo "ERROR: mirrorlist3 did not restart correctly" - exit 1 -fi - -echo "enable server mirror-lists-backend/mirrorlist-local3" | nc -U /var/run/haproxy-admin >& /dev/null -{% endif %} diff --git a/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml b/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml index 607f0d9348..7fae8a9337 100644 --- a/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml +++ b/roles/mirrormanager/mirrorlist_proxy/tasks/main.yml @@ -1,5 +1,5 @@ --- -# tasklist for setting up the mirrorlist app as a container on proxies +# tasklist for setting up the mirrorlist app on the proxies - name: add mirrormanager user - uid {{ mirrormanager_uid }} @@ -7,30 +7,6 @@ tags: - mirrorlist_proxy -- name: add mirrormanager subdirectories - file: dest="{{item}}" mode=0755 state=directory owner=mirrormanager group=mirrormanager - with_items: - - /home/mirrormanager/rundir - - /home/mirrormanager/rundir/libpod - tags: - - mirrorlist_proxy - -- name: mirrormanager subgid - lineinfile: - dest: /etc/subgid - regexp: "^mirrormanager:" - line: "mirrormanager:{{ mirrormanager_subgid }}" - tags: - - mirrorlist_proxy - -- name: mirrormanager subuid - lineinfile: - dest: /etc/subuid - regexp: "^mirrormanager:" - line: "mirrormanager:{{ mirrormanager_subuid }}" - tags: - - mirrorlist_proxy - # mirrormanager user ssh key(s) - name: add authorized_keys for mirrormanager authorized_key: key="{{ item }}" user=mirrormanager state=present @@ -46,7 +22,6 @@ - /srv/mirrorlist/data - /srv/mirrorlist/data/mirrorlist1 - /srv/mirrorlist/data/mirrorlist2 - - /srv/mirrorlist/data/mirrorlist3 - /var/log/mirrormanager tags: - mirrorlist_proxy @@ -57,7 +32,6 @@ - /srv/mirrorlist/data - /srv/mirrorlist/data/mirrorlist1 - /srv/mirrorlist/data/mirrorlist2 - - /srv/mirrorlist/data/mirrorlist3 tags: - mirrorlist_proxy @@ -84,23 +58,6 @@ tags: - mirrorlist_proxy -- name: Ensure log file for content exists - file: dest="{{item}}" owner=mirrormanager group=mirrormanager mode=0755 state=touch - with_items: - - /var/log/mirrormanager/mirrorlist3.service.log - when: datacenter == 'iad2' - tags: - - mirrorlist_proxy - -- name: for the rust based mirrorlist server chown log files - file: dest="{{item}}" owner=mirrormanager group=mirrormanager - with_items: - - /var/log/mirrormanager/mirrorlist3.service.log - when: datacenter == 'iad2' - tags: - - mirrorlist_proxy - - - name: set logrotate_read_inside_containers so logrotate works seboolean: name=logrotate_read_inside_containers state=yes persistent=yes tags: @@ -111,27 +68,13 @@ tags: - mirrorlist_proxy -- name: Install podman - package: name={{item}} state=present - with_items: - - podman - tags: - - mirrorlist_proxy - -- name: Pull mirrorlist image - command: podman pull -q "{{ mirrorlist_container_image }}" - tags: - - mirrorlist_proxy - check_mode: yes - - name: install mirrorlist-server package - package: name={{ item }} state=present enablerepo=epel-testing + package: name={{ item }} state=present with_items: - mirrorlist-server tags: - packages - mirrorlist_proxy - when: ansible_distribution_major_version|int >= 31 # We deploy two service files. Both listen on a different port, so that we can switch # them out as part of the protobuf cache deployment without having any local downtime. @@ -145,18 +88,6 @@ notify: - reload systemd -# We deploy two service files. Both listen on a different port, so that we can switch -# them out as part of the protobuf cache deployment without having any local downtime. -- name: Deploy service files for m3 - template: src=mirrorlist.service.j2 dest=/etc/systemd/system/mirrorlist{{ item }}.service - with_items: - - 3 - tags: - - mirrorlist_proxy - notify: - - reload systemd - when: datacenter == 'iad2' - # enable both of them to run on boot - name: Enable mirrorlist1 service: name=mirrorlist1 enabled=yes @@ -168,30 +99,24 @@ tags: - mirrorlist_proxy -- name: Enable mirrorlist3 - service: name=mirrorlist3 enabled=yes - tags: - - mirrorlist_proxy - when: datacenter == 'iad2' - # install our cron script to handle hourly new protbuf cache changes. -- name: install script to restart mirrorlist containers on protobuf cache changes - template: src=restart-mirrorlist-containers.j2 dest=/usr/local/bin/restart-mirrorlist-containers mode=0755 +- name: install script to restart mirrorlist processes on protobuf cache changes + copy: src=restart-mirrorlist-processes dest=/usr/local/bin/restart-mirrorlist-processes mode=0755 tags: - mirrorlist_proxy - name: Setup hourly cron at for mirrorlist restarts - cron: name="restart-mirrorlist-containers" minute="20" user="root" - job="/usr/local/bin/restart-mirrorlist-containers" - cron_file=restart-mirrorlist-containers + cron: name="restart-mirrorlist-processes" minute="20" user="root" + job="/usr/local/bin/restart-mirrorlist-processes" + cron_file=restart-mirrorlist-processes tags: - mirrorlist_proxy -- name: Set cron MAILTO for restart-mirrorlist-containers +- name: Set cron MAILTO for restart-mirrorlist-processes cronvar: name: MAILTO value: "root@fedoraproject.org,adrian@lisas.de" - cron_file: restart-mirrorlist-containers + cron_file: restart-mirrorlist-processes tags: - mirrorlist_proxy diff --git a/roles/mirrormanager/mirrorlist_proxy/templates/mirrorlist.service.j2 b/roles/mirrormanager/mirrorlist_proxy/templates/mirrorlist.service.j2 index 9653d546b6..af25879a36 100644 --- a/roles/mirrormanager/mirrorlist_proxy/templates/mirrorlist.service.j2 +++ b/roles/mirrormanager/mirrorlist_proxy/templates/mirrorlist.service.j2 @@ -3,27 +3,6 @@ Description=Mirrorlist Server {{ item }} [Service] User=mirrormanager -{% if ansible_distribution_major_version|int < 31 %} -ExecStartPre=-/usr/bin/podman stop -t 1 %n -ExecStartPre=-/usr/bin/podman rm %n --force -ExecStart=/usr/bin/podman run \ - --rm=true \ - --net=host --userns=keep-id \ - --rm=true --name %n \ - -v /usr/share/GeoIP:/usr/share/GeoIP \ - -v /srv/mirrorlist/data/mirrorlist{{ item }}:/var/lib/mirrormanager \ - -v /var/log/mirrormanager:/var/log/mirrormanager \ -{% if env == "staging" %} - {{ mirrorlist_container_image_stg }} \ -{% else %} - {{ mirrorlist_container_image }} \ -{% endif %} - --port 1808{{ item }} \ - --listen 127.0.0.1 \ - -l /var/log/mirrormanager/%n.log -ExecStop=/usr/bin/podman stop -t 1 %n -KillMode=none -{% else %} ExecStart=/usr/bin/mirrorlist-server \ --port 1808{{ item }} \ --listen 127.0.0.1 \ @@ -32,7 +11,6 @@ ExecStart=/usr/bin/mirrorlist-server \ --internet2_netblocks /srv/mirrorlist/data/mirrorlist{{ item }}/i2_netblocks.txt \ --global_netblocks /srv/mirrorlist/data/mirrorlist{{ item }}/global_netblocks.txt \ --cccsv /srv/mirrorlist/data/mirrorlist{{ item }}/country_continent.csv -{% endif %} [Install] WantedBy=multi-user.target diff --git a/roles/mirrormanager/mirrorlist_proxy/vars/main.yml b/roles/mirrormanager/mirrorlist_proxy/vars/main.yml index 398f8c037b..089784c9b0 100644 --- a/roles/mirrormanager/mirrorlist_proxy/vars/main.yml +++ b/roles/mirrormanager/mirrorlist_proxy/vars/main.yml @@ -1,6 +1,4 @@ mirrormanager_uid: 441 mirrormanager_gid: 441 -mirrormanager_subuid: "100000:65536" -mirrormanager_subgid: "100000:65536" mirrors_gid: 263 mirrors2_gid: 529