s3-mirror: drop this role as it's no longer used

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
This commit is contained in:
Kevin Fenzi 2024-07-03 15:22:57 -07:00
parent 432a3a497b
commit 7bfb291a9a
7 changed files with 0 additions and 469 deletions

View file

@ -1,7 +0,0 @@
[default]
region = us-west-2
output = text
s3 =
max_concurrent_requests = 50
multipart_threshold = 64MB
multipart_chunksize = 16MB

View file

@ -1,72 +0,0 @@
#!/usr/bin/env bash
# (c) 2019 Red Hat, Inc.
# LGPL
# Author: Rick Elrod <relrod@redhat.com>
if [[ "$1" == "" ]] || [[ $1 != /pub* ]] || [[ $1 != */ ]]; then
echo "Syntax: $0 /pub/path/to/sync/"
echo "NOTE! Path must end with a trailing /"
exit 1
fi
aws_sync=( aws s3 sync --no-follow-symlinks )
# first run do not delete anything or copy the repodata.
exclude=(
--exclude "*.snapshot/*"
--exclude "*source/*"
--exclude "*SRPMS/*"
--exclude "*debug/*"
--exclude "*beta/*"
--exclude "*ppc/*"
--exclude "*ppc64/*"
--exclude "*repoview/*"
--exclude "*Fedora/*"
--exclude "*EFI/*"
--exclude "*core/*"
--exclude "*extras/*"
--exclude "*LiveOS/*"
--exclude "*development/rawhide/*"
--only-show-errors
)
S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
DIST_ID=E2KJMDC0QAJDMU
MAX_CACHE_SEC=60
DNF_GENTLY_TIMEOUT=120
# First run this command that syncs, but does not delete.
# It also excludes repomd.xml.
CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" )
# Next we run this command which syncs repomd.xml files. Include must precede
# the large set of excludes. Make sure that the 'max-age' isn't too large so
# we know that we can start removing old data ASAP.
CMD2=( "${aws_sync[@]}" --exclude "*" --include "*/repomd.xml" "${excludes[@]}"
--cache-control "max-age=$MAX_CACHE_SEC" )
# Then we delete old RPMs and old metadata (but after invalidating caches).
CMD3=( "${aws_sync[@]}" "${excludes[@]}" --delete )
#echo "$CMD /srv$1 s3://s3-mirror-us-west-1-02.fedoraproject.org$1"
echo "Starting $1 sync at $(date)" >> /var/log/s3-mirror/timestamps
"${CMD1[@]}" "/srv$1" "s3://$S3_MIRROR$1"
"${CMD2[@]}" "/srv$1" "s3://$S3_MIRROR$1"
# Always do the invalidations because they are quick and prevent issues
# depending on which path is synced.
for file in $(echo $1/repodata/repomd.xml ); do
aws cloudfront create-invalidation --distribution-id $DIST_ID --paths "$file" > /dev/null
done
SLEEP=$(( MAX_CACHE_SEC + DNF_GENTLY_TIMEOUT ))
echo "Ready $1 sync, giving dnf downloads ${SLEEP}s before delete, at $(date)" >> /var/log/s3-mirror/timestamps
# Consider some DNF processes started downloading metadata before we invalidated
# caches, and started with outdated repomd.xml file. Give it few more seconds
# so they have chance to download the rest of metadata and RPMs.
sleep $SLEEP
"${CMD3[@]}" "/srv$1" "s3://$S3_MIRROR$1"
echo "Ending $1 sync at $(date)" >> /var/log/s3-mirror/timestamps

View file

@ -1,146 +0,0 @@
#!/usr/bin/env bash
# (c) 2018 Red Hat, Inc.
# LGPL
# Author: Rick Elrod <relrod@redhat.com>
aws_sync=( aws s3 sync --no-follow-symlinks )
excludes=(
--exclude "*/.snapshot/*"
--exclude "*/source/*"
--exclude "*/SRPMS/*"
--exclude "*/debug/*"
--exclude "*/beta/*"
--exclude "*/ppc/*"
--exclude "*/ppc64/*"
--exclude "*/repoview/*"
--exclude "*/Fedora/*"
--exclude "*/EFI/*"
--exclude "*/core/*"
--exclude "*/extras/*"
--exclude "*/LiveOS/*"
--exclude "*/development/rawhide/*"
--exclude "*/releases/8/*"
--exclude "*/releases/9/*"
--exclude "*/releases/10/*"
--exclude "*/releases/11/*"
--exclude "*/releases/12/*"
--exclude "*/releases/13/*"
--exclude "*/releases/14/*"
--exclude "*/releases/15/*"
--exclude "*/releases/16/*"
--exclude "*/releases/17/*"
--exclude "*/releases/18/*"
--exclude "*/releases/19/*"
--exclude "*/releases/20/*"
--exclude "*/releases/21/*"
--exclude "*/releases/22/*"
--exclude "*/releases/23/*"
--exclude "*/releases/24/*"
--exclude "*/releases/25/*"
--exclude "*/releases/26/*"
--exclude "*/releases/27/*"
--exclude "*/releases/28/*"
--exclude "*/releases/29/*"
--exclude "*/updates/8/*"
--exclude "*/updates/9/*"
--exclude "*/updates/10/*"
--exclude "*/updates/11/*"
--exclude "*/updates/12/*"
--exclude "*/updates/13/*"
--exclude "*/updates/14/*"
--exclude "*/updates/15/*"
--exclude "*/updates/16/*"
--exclude "*/updates/17/*"
--exclude "*/updates/18/*"
--exclude "*/updates/19/*"
--exclude "*/updates/20/*"
--exclude "*/updates/21/*"
--exclude "*/updates/22/*"
--exclude "*/updates/23/*"
--exclude "*/updates/24/*"
--exclude "*/updates/25/*"
--exclude "*/updates/26/*"
--exclude "*/updates/27/*"
--exclude "*/updates/28/*"
--exclude "*/updates/29/*"
--exclude "*/updates/testing/8/*"
--exclude "*/updates/testing/9/*"
--exclude "*/updates/testing/10/*"
--exclude "*/updates/testing/11/*"
--exclude "*/updates/testing/12/*"
--exclude "*/updates/testing/13/*"
--exclude "*/updates/testing/14/*"
--exclude "*/updates/testing/15/*"
--exclude "*/updates/testing/16/*"
--exclude "*/updates/testing/17/*"
--exclude "*/updates/testing/18/*"
--exclude "*/updates/testing/19/*"
--exclude "*/updates/testing/20/*"
--exclude "*/updates/testing/21/*"
--exclude "*/updates/testing/22/*"
--exclude "*/updates/testing/23/*"
--exclude "*/updates/testing/24/*"
--exclude "*/updates/testing/25/*"
--exclude "*/updates/testing/26/*"
--exclude "*/updates/testing/27/*"
--exclude "*/updates/testing/28/*"
--exclude "*/updates/testing/29/*"
)
S3_MIRROR=s3-mirror-us-west-1-02.fedoraproject.org
DIST_ID=E2KJMDC0QAJDMU
MAX_CACHE_SEC=60
DNF_GENTLY_TIMEOUT=120
# First run this command that syncs, but does not delete.
# It also excludes repomd.xml.
CMD1=( "${aws_sync[@]}" "${excludes[@]}" --exclude "*/repomd.xml" )
# Next we run this command which syncs repomd.xml files. Include must precede
# the large set of excludes. Make sure that the 'max-age' isn't too large so
# we know that we can start removing old data ASAP.
CMD2=( "${aws_sync[@]}" --exclude "*" --include "*/repomd.xml" "${excludes[@]}"
--cache-control "max-age=$MAX_CACHE_SEC" )
# Then we delete old RPMs and old metadata (but after invalidating caches).
CMD3=( "${aws_sync[@]}" "${excludes[@]}" --delete )
# Sync EPEL
#echo $CMD /srv/pub/epel/ s3://$S3_MIRROR/pub/epel/
echo "Starting EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
"${CMD1[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
"${CMD2[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
echo "Ending EPEL sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/epel/6/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
for file in $(echo /srv/pub/epel/7/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
for file in $(echo /srv/pub/epel/8/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
# Sync Fedora
#echo $CMD /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/
echo "Starting Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
"${CMD1[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
"${CMD2[@]}" /srv/pub/fedora/ "s3://$S3_MIRROR/pub/fedora/"
echo "Ending Fedora sync at $(date)" >> /var/log/s3-mirror/timestamps
for file in $(echo /srv/pub/fedora/linux/updates/*/*/*/repodata/repomd.xml | sed 's#/srv##g'); do
aws cloudfront create-invalidation --distribution-id "$DIST_ID" --paths "$file"
done
SLEEP=$(( MAX_CACHE_SEC + DNF_GENTLY_TIMEOUT ))
# Consider some DNF processes started downloading metadata before we invalidated
# caches, and started with outdated repomd.xml file. Give it few more seconds
# so they have chance to download the rest of metadata and RPMs.
sleep $SLEEP
"${CMD3[@]}" /srv/pub/epel/ "s3://$S3_MIRROR/pub/epel/"
"${CMD3[@]}" /srv/pub/fedora/ s3://$S3_MIRROR/pub/fedora/

View file

@ -1,33 +0,0 @@
#!/bin/sh
s3cmd=/usr/bin/s3cmd
S3CMD_ARGS="sync \
--recursive \
--delete-removed \
--verbose \
"
targets="s3-mirror-us-east-1 s3-mirror-us-west-1 s3-mirror-us-west-2 s3-mirror-eu-west-1 s3-mirror-ap-northeast-1"
for t in ${targets}; do
mkdir -p /var/log/s3-mirror-logs/bucket/${t}
done
echo "=============================================================="
echo -n "Starting at "
date
time $s3cmd $S3CMD_ARGS s3://s3-mirror-logs.fedoraproject.org/s3-mirror-us-east-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-east-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-1.fedoraproject.org/s3-mirror-us-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-us-west-2.fedoraproject.org/s3-mirror-us-west-2/ /var/log/s3-mirror-logs/bucket/s3-mirror-us-west-2/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-eu-west-1.fedoraproject.org/s3-mirror-eu-west-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-eu-west-1/ &
time $s3cmd $S3CMD_ARGS s3://aws-logs-ap-northeast-1.fedoraproject.org/s3-mirror-s3-northeast-1/ /var/log/s3-mirror-logs/bucket/s3-mirror-ap-northeast-1/ &
wait
# concatenate them for processing
rm /var/log/s3-mirror-logs/combined.log
find /var/log/s3-mirror-logs/bucket/ -type f -mtime -2 | sort -n | while read f; do
cat $f >> /var/log/s3-mirror-logs/combined.log
done
echo -n "Ending at "
date

View file

@ -1,148 +0,0 @@
- name: Include vars
include_vars: dir=/srv/web/infra/ansible/vars/all/ ignore_files=README
tags:
- s3-mirror
- name: Create s3-mirror group
group: gid=442 name=s3-mirror
tags:
- s3-mirror
- name: Create s3-mirror user
user: group=s3-mirror name=s3-mirror comment="s3-mirror user" home=/home/s3-mirror shell=/bin/bash
tags:
- s3-mirror
- name: Create needed directories
file: path={{item}} owner=s3-mirror group=s3-mirror state=directory mode=0755
with_items:
- /var/log/s3-mirror
- /var/lib/s3-mirror
- /var/run/s3-mirror
- /home/s3-mirror/.aws
#- /var/log/s3-mirror-logs
#- /var/log/s3-mirror-logs/bucket
tags:
- s3-mirror
- name: aws credentials file
template: src=credentials dest=/home/s3-mirror/.aws/credentials owner=s3-mirror group=s3-mirror mode=0600
tags:
- s3-mirror
- name: aws config file
copy: src=config dest=/home/s3-mirror/.aws/config owner=s3-mirror group=s3-mirror mode=0600
tags:
- s3-mirror
- name: Install packages
package: name={{ item }} state=present
with_items:
- awscli
# - mirrormanager2-client
tags:
- s3-mirror
- name: s3 script
copy: src={{item}} dest=/usr/local/bin/{{item}} owner=s3-mirror group=s3-mirror mode=0755
with_items:
- s3.sh
- s3-sync-path.sh
tags:
- s3-mirror
- name: s3sync cron main that takes forever
cron: name="s3sync-main" minute="0" hour="0" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-main /usr/local/bin/s3.sh 2>&1 | /usr/local/bin/nag-once s3.sh 1d 2>&1'
cron_file=s3.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - test releases
cron: name="s3sync-updates-current" minute="40" hour="5,9,13,19" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-test-releases "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/releases/test/" 2>&1 | /usr/local/bin/nag-once s3-test-releases.sh 1d 2>&1'
cron_file=s3-test-releases.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - updates for current
cron: name="s3sync-updates-current" minute="0" hour="3,9,15,21" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-current "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int }}/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-updates-current.sh 1d 2>&1'
cron_file=s3-updates-current.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - updates for development/current+1 x86_64
cron: name="s3sync-updates-current" minute="0" hour="2,7,10" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/x86_64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-x86_64.sh 1d 2>&1'
cron_file=s3-updates-dev-cur-plus-1-x86_64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - updates for development/current+1 aarch64
cron: name="s3sync-updates-current" minute="0" hour="4,11,18" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-dev-cur-plus-1-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/development/{{ FedoraCycleNumber|int + 1 }}/Everything/aarch64/os/" 2>&1 | /usr/local/bin/nag-once s3-updates-dev-cur-plus-1-aarch64.sh 1d 2>&1'
cron_file=s3-updates-dev-cur-plus-1-aarch64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - updates for current-1
cron: name="s3sync-updates-previous" minute="30" hour="0,6,12,18" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-updates-previous "/usr/local/bin/s3-sync-path.sh /pub/fedora/linux/updates/{{ FedoraCycleNumber|int - 1 }}/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-updates-previous.sh 1d 2>&1'
cron_file=s3-updates-previous.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - epel 7 x86_64
cron: name="s3sync-epel7-x86_64" minute="10" hour="2,5,8,11,14,17,20,23" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-epel7-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-x86_64.sh 1d 2>&1'
cron_file=s3-epel7-x86_64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - epel 7 aarch64
cron: name="s3sync-epel7-aarch64" minute="20" hour="4,7,10,13,16,19,22" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-epel7-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/7/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel7-aarch64.sh 1d 2>&1'
cron_file=s3-epel7-aarch64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - epel 8 Everything x86_64
cron: name="s3sync-epel8-everything-x86_64" minute="43" hour="3,6,9,12,15,17,20,23" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-x86_64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/x86_64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-x86_64.sh 1d 2>&1'
cron_file=s3-epel8-everything-x86_64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
- name: s3sync cron - epel 8 Everything aarch64
cron: name="s3sync-epel8-everything-aarch64" minute="38" hour="4,7,10,13,16,19,22" user="s3-mirror"
job='/usr/local/bin/lock-wrapper s3sync-epel8-everything-aarch64 "/usr/local/bin/s3-sync-path.sh /pub/epel/8/Everything/aarch64/" 2>&1 | /usr/local/bin/nag-once s3-epel8-everything-aarch64.sh 1d 2>&1'
cron_file=s3-epel8-everything-aarch64.sh
disabled=true
when: env != 'staging' and inventory_hostname.startswith('mm-backend01.')
tags:
- s3-mirror
#- name: s3sync-logs cron
# cron: name="s3sync-logs" hour="0" user="root"
# job='/usr/local/bin/lock-wrapper s3sync-logs "/bin/sleep $((${RANDOM} \% 300)); /usr/local/bin/s3sync-logs >> /var/log/s3-mirror-logs/s3sync-logs.log 2>&1" | /usr/local/bin/nag-once s3mirror-logs 12h 2>&1'
# cron_file=s3sync-logs
# state=absent

View file

@ -1,3 +0,0 @@
[default]
aws_access_key_id = {{ s3_mirror_aws_access_key }}
aws_secret_access_key = {{ s3_mirror_aws_secret_key }}

View file

@ -1,60 +0,0 @@
[global]
# if enabled=0, no data is sent to the database
enabled=1
# server= is the URL to the MirrorManager XML-RPC interface
server=https://admin.fedoraproject.org/mirrormanager/xmlrpc
[site]
# if enabled=0, no data about this site is sent to the database
enabled=1
# Name and Password fields need to match the Site name and password
# fields you entered for your Site in the MirrorManager database at
# https://admin.fedoraproject.org/mirrormanager
name={{ s3_mirror_report_mirror_site }}
password={{ s3_mirror_report_mirror_pw }}
[host]
# if enabled=0, no data about this host is sent to the database
enabled=1
# Name field need to match the Host name field you entered for your
# Host in the MirrorManager database at
# https://admin.fedoraproject.org/mirrormanager
name={{ item }}.fedoraproject.org
# if user_active=0, no data about this category is given to the public
# This can be used to toggle between serving and not serving data,
# such enabled during the nighttime (when you have more idle bandwidth
# available) and disabled during the daytime.
# By not specifying user_active, the database will not be updated.
# user_active=1
[stats]
# Stats are only sent when run with the -s option
# and when this section is enabled.
# This feature is not presently implemented
enabled=0
apache=/var/log/httpd/access_log
vsftpd=/var/log/vsftpd.log
# remember to enable log file and transfer logging in rsyncd.conf
rsyncd=/var/log/rsyncd.log
# Content Categories
# These sections match the Categories for content tracked by MirrorManager.
#
# enabled=1 means information about this category will be sent to the database.
# enabled=0, no data about this host is sent to the database. If the
# database already has information for you for this Category, it will
# remain unchanged. This can be used to update the database after you
# have manually synced some infrequently-updated content, such as
# historical releases.
#
# path= is the path on your local disk to the top-level directory for this Category
[Fedora Linux]
enabled=0
path=/srv/pub/fedora/linux
[Fedora EPEL]
enabled=1
path=/srv/pub/epel