MM: add cron jobs to generate the mirrorlist
Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard
parent
5bd9446dfe
commit
3dbfcfcf84
7 changed files with 187 additions and 2 deletions
|
|
@ -127,6 +127,10 @@
|
|||
app: mirrormanager
|
||||
template: cron-accesses.yml
|
||||
objectname: cron-accesses.yml
|
||||
- role: openshift/object
|
||||
app: mirrormanager
|
||||
template: cron-mirrorlist-cache.yml
|
||||
objectname: cron-mirrorlist-cache.yml
|
||||
|
||||
- role: openshift/start-build
|
||||
app: mirrormanager
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@
|
|||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: data
|
||||
name: mirrorlist-cache
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
storage: 2Gi
|
||||
storageClassName: ocs-storagecluster-cephfs
|
||||
# ---
|
||||
# apiVersion: v1
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ items:
|
|||
{{ load_file('mirrormanager2.cfg.py') | indent(6) }}
|
||||
gunicorn.conf.py: |-
|
||||
{{ load_file('gunicorn.conf.py') | indent(6) }}
|
||||
generate-mirrorlist-cache.cfg: |-
|
||||
{{ load_file('generate-mirrorlist-cache.cfg') | indent(6) }}
|
||||
- apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
|
|
@ -34,3 +36,5 @@ items:
|
|||
data:
|
||||
create_statistics.sh: |-
|
||||
{{ load_file('create_statistics.sh') | indent(6) }}
|
||||
update-mirrorlist-cache.sh: |-
|
||||
{{ load_file('update-mirrorlist-cache.sh') | indent(6) }}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ OUTPUT=`mktemp -d`
|
|||
|
||||
trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
for proxy in ${MIRRORLIST_PROXIES}; do
|
||||
|
|
|
|||
|
|
@ -0,0 +1,146 @@
|
|||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: update-mirrorlist-cache
|
||||
spec:
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
concurrencyPolicy: Forbid
|
||||
schedule: "0,15,30,45 * * * *"
|
||||
startingDeadlineSeconds: 500
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: mirrormanager
|
||||
image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
|
||||
command: ["bash", "/opt/scripts/update-mirrorlist-cache.sh"]
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: "/etc/mirrormanager"
|
||||
readOnly: true
|
||||
- name: scripts
|
||||
mountPath: "/opt/scripts"
|
||||
readOnly: true
|
||||
- name: ssh-key
|
||||
mountPath: /etc/mirrormanager-ssh/ssh_mirrorlist_proxies.key
|
||||
subPath: ssh_mirrorlist_proxies.key
|
||||
readOnly: true
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: config
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: scripts
|
||||
- name: ssh-key
|
||||
secret:
|
||||
secretName: ssh-mirrorlist-proxies-key
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: mirrorlist-cache
|
||||
---
|
||||
# Sync netblocks list once a day
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: netblocks-global
|
||||
spec:
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
concurrencyPolicy: Forbid
|
||||
schedule: "30 0 * * *"
|
||||
startingDeadlineSeconds: 500
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: mirrormanager
|
||||
image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
|
||||
command: ["/opt/app-root/bin/mm2_get-netblocks", "global", "-o", "/data/global_netblocks.txt"]
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: "/etc/mirrormanager"
|
||||
readOnly: true
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: mirrorlist-cache
|
||||
---
|
||||
# Run the script for Internet2 netblocks very late to
|
||||
# make sure the files are all there. The script is
|
||||
# looking in date dependent directories and depending
|
||||
# on the timezone of the remote server the files appear late.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: netblocks-internet2
|
||||
spec:
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
concurrencyPolicy: Forbid
|
||||
schedule: "0 23 * * *"
|
||||
startingDeadlineSeconds: 500
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: mirrormanager
|
||||
image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
|
||||
command: ["/opt/app-root/bin/mm2_get-netblocks", "internet2", "-o", "/data/i2_netblocks.txt"]
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: "/etc/mirrormanager"
|
||||
readOnly: true
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: config
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: mirrorlist-cache
|
||||
---
|
||||
# Amazon EC2 netblocks
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: netblocks-internet2
|
||||
spec:
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
concurrencyPolicy: Forbid
|
||||
schedule: "30 1 * * *"
|
||||
startingDeadlineSeconds: 500
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: mirrormanager
|
||||
image: image-registry.openshift-image-registry.svc:5000/mirrormanager/mirrormanager2:latest
|
||||
command: ["/opt/app-root/bin/mm2_update-EC2-netblocks"]
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: "/etc/mirrormanager"
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: config
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{% if env == 'staging' %}
|
||||
DB_URL="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}"
|
||||
{% else %}
|
||||
DB_URL="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
|
||||
{% endif %}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
|
||||
MM_ROOT=/opt/app-root/src
|
||||
CACHEDIR=/data
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
rm -rf ${CACHEDIR}/old
|
||||
mkdir -p ${CACHEDIR}/old
|
||||
cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/ 2>/dev/null
|
||||
|
||||
${MM_ROOT}/bin/generate-mirrorlist-cache -o ${CACHEDIR}/mirrorlist_cache.proto
|
||||
|
||||
# Update the files on the proxies
|
||||
{% if env == 'production' %}
|
||||
for server in ${MIRRORLIST_PROXY}; do
|
||||
rsync -az --delete-delay --delay-updates --delete \
|
||||
${MM_ROOT}/mirrormanager2/crawler/country_continent.csv \
|
||||
${CACHEDIR}/mirrorlist_cache.proto \
|
||||
${CACHEDIR}/*.txt \ # Netblocks
|
||||
${server}:/srv/mirrorlist/data/mirrorlist1/ &
|
||||
done
|
||||
{% endif %}
|
||||
Loading…
Add table
Add a link
Reference in a new issue