mirrormanager: retire vm's now that it's moved to openshift
Lets retire these rhel7 vm's from ansible/running. I will be saving off the disks and xml for all of the vm's, so in the event we need to bring something back or look at something, we can do so. Signed-off-by: Kevin Fenzi <kevin@scrye.com>
This commit is contained in:
Kevin Fenzi
parent
9064d0c26b
commit
e838841868
59 changed files with 0 additions and 2262 deletions
|
|
@ -1,15 +0,0 @@
|
|||
---
|
||||
# Define resources for this group of hosts here.
|
||||
deployment_type: prod
|
||||
ipa_client_shell_groups:
|
||||
- sysadmin-noc
|
||||
- sysadmin-veteran
|
||||
- sysadmin-web
|
||||
ipa_client_sudo_groups:
|
||||
- sysadmin-noc
|
||||
- sysadmin-web
|
||||
ipa_host_group: mirrormanager
|
||||
ipa_host_group_desc: Mirror Manager
|
||||
mirrormanager_db_host: 'db01'
|
||||
mm2_checkin: false
|
||||
primary_auth_source: ipa
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager backend cron tasks
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- alias: mirrormanager
|
||||
can_send:
|
||||
- mirrormanager.netblocks.get
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: mirrormanager
|
||||
service: shell
|
||||
mem_size: 6144
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager backend cron tasks
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- can_send:
|
||||
- mirrormanager.netblocks.get
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: mirrormanager
|
||||
service: shell
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager crawlers
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- can_send:
|
||||
- mirrormanager.crawler.complete
|
||||
- mirrormanager.crawler.start
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: mirrormanager
|
||||
service: shell
|
||||
rsyncd_conf: "rsyncd.conf.crawler"
|
||||
tcp_ports: [873]
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager crawlers
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- can_send:
|
||||
- mirrormanager.crawler.complete
|
||||
- mirrormanager.crawler.start
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: mirrormanager
|
||||
service: shell
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager frontend WSGI app
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- can_send:
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: root
|
||||
service: shell
|
||||
- group: apache
|
||||
owner: root
|
||||
service: mirrormanager2
|
||||
mem_size: 4096
|
||||
tcp_ports: [80,
|
||||
# These 2 ports are used by fedmsg.
|
||||
# One for each wsgi thread.
|
||||
3000, 3001]
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora admin - admin@fedoraproject.org
|
||||
csi_purpose: Run mirrormanager frontend WSGI app
|
||||
csi_relationship: |
|
||||
TODO - we should document:
|
||||
|
||||
* what kinds of processes run here
|
||||
* what other services they depend on
|
||||
* what other services depend on it
|
||||
# For the MOTD
|
||||
csi_security_category: Moderate
|
||||
fedmsg_certs:
|
||||
- can_send:
|
||||
- logger.log
|
||||
group: sysadmin
|
||||
owner: root
|
||||
service: shell
|
||||
- group: apache
|
||||
owner: root
|
||||
service: mirrormanager2
|
||||
tcp_ports: [80,
|
||||
# These 2 ports are used by fedmsg.
|
||||
# One for each wsgi thread.
|
||||
3000, 3001]
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
---
|
||||
# Define resources for this group of hosts here.
|
||||
deployment_type: stg
|
||||
ipa_client_shell_groups:
|
||||
- sysadmin-noc
|
||||
- sysadmin-veteran
|
||||
- sysadmin-web
|
||||
ipa_client_sudo_groups:
|
||||
- sysadmin-noc
|
||||
- sysadmin-web
|
||||
ipa_host_group: mirrormanager
|
||||
ipa_host_group_desc: Mirror Manager
|
||||
mirrormanager_db_host: 'db01.stg'
|
||||
mm2_checkin: false
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.163.254
|
||||
eth0_ipv4_ip: 10.3.163.60
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
max_mem_size: 16384
|
||||
mem_size: 16384
|
||||
# nfs mount options, overrides the all/default
|
||||
nfs_mount_opts: "ro,hard,bg,intr,nodev,nosuid,nfsvers=3"
|
||||
num_cpus: 2
|
||||
vmhost: vmhost-x86-01.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.166.254
|
||||
eth0_ipv4_ip: 10.3.166.25
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
max_mem_size: 16384
|
||||
mem_size: 16384
|
||||
# nfs mount options, overrides the all/default
|
||||
nfs_mount_opts: "ro,hard,bg,intr,nodev,nosuid,nfsvers=3"
|
||||
num_cpus: 2
|
||||
vmhost: vmhost-x86-12.stg.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.166.254
|
||||
eth0_ipv4_ip: 10.3.166.69
|
||||
external: false
|
||||
ks_repo: http://10.3.163.35/pub/fedora/linux/releases/39/Server/x86_64/os/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-fedora
|
||||
vmhost: vmhost-x86-11.stg.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.163.254
|
||||
eth0_ipv4_ip: 10.3.163.62
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
mem_size: 40960
|
||||
num_cpus: 4
|
||||
vmhost: vmhost-x86-02.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.166.254
|
||||
eth0_ipv4_ip: 10.3.166.26
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
max_mem_size: 65536
|
||||
mem_size: 40960
|
||||
num_cpus: 4
|
||||
vmhost: vmhost-x86-11.stg.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.163.254
|
||||
eth0_ipv4_ip: 10.3.163.96
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
mem_size: 40960
|
||||
num_cpus: 4
|
||||
vmhost: vmhost-x86-03.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
csi_primary_contact: Fedora Admins - admin@fedoraproject.org
|
||||
csi_purpose: MirrorManager Checkin endpoint
|
||||
csi_relationship: |
|
||||
Has a very restricted set of in/out communication allowed, due to
|
||||
special circumstances. For details, ask puiterwijk.
|
||||
csi_security_category: High
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.163.254
|
||||
eth0_ipv4_ip: 10.3.163.91
|
||||
fedmsg_certs: []
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
mem_size: 8192
|
||||
mm2_checkin: true
|
||||
num_cpus: 2
|
||||
tcp_ports: [80, 443]
|
||||
vmhost: vmhost-x86-04.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.163.254
|
||||
eth0_ipv4_ip: 10.3.163.61
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
mem_size: 8192
|
||||
num_cpus: 2
|
||||
tcp_ports: [80, 443]
|
||||
vmhost: vmhost-x86-03.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
---
|
||||
datacenter: iad2
|
||||
eth0_ipv4_gw: 10.3.166.254
|
||||
eth0_ipv4_ip: 10.3.166.27
|
||||
ks_repo: http://10.3.163.35/repo/rhel/RHEL7-x86_64/
|
||||
ks_url: http://10.3.163.35/repo/rhel/ks/kvm-rhel-7-iad2
|
||||
lvm_size: 20000
|
||||
mem_size: 8192
|
||||
num_cpus: 2
|
||||
tcp_ports: [80, 443]
|
||||
vmhost: vmhost-x86-08.stg.iad2.fedoraproject.org
|
||||
volgroup: /dev/vg_guests
|
||||
|
|
@ -289,37 +289,6 @@ proxy40.fedoraproject.org
|
|||
proxy101.iad2.fedoraproject.org
|
||||
proxy110.iad2.fedoraproject.org
|
||||
|
||||
[mm_frontend]
|
||||
mm-frontend01.iad2.fedoraproject.org
|
||||
mm-frontend-checkin01.iad2.fedoraproject.org
|
||||
|
||||
[mm_backend]
|
||||
mm-backend01.iad2.fedoraproject.org
|
||||
|
||||
[mm_crawler]
|
||||
mm-crawler01.iad2.fedoraproject.org
|
||||
mm-crawler02.iad2.fedoraproject.org
|
||||
|
||||
[mm_frontend_stg]
|
||||
mm-frontend01.stg.iad2.fedoraproject.org
|
||||
|
||||
[mm_backend_stg]
|
||||
mm-backend01.stg.iad2.fedoraproject.org
|
||||
|
||||
[mm_crawler_stg]
|
||||
mm-crawler01.stg.iad2.fedoraproject.org
|
||||
mm-crawler-dev.stg.iad2.fedoraproject.org
|
||||
|
||||
[mm:children]
|
||||
mm_frontend
|
||||
mm_backend
|
||||
mm_crawler
|
||||
|
||||
[mm_stg:children]
|
||||
mm_frontend_stg
|
||||
mm_backend_stg
|
||||
mm_crawler_stg
|
||||
|
||||
[people]
|
||||
people01.fedoraproject.org
|
||||
|
||||
|
|
|
|||
|
|
@ -1,112 +0,0 @@
|
|||
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml"
|
||||
vars:
|
||||
myhosts: "mm:mm_stg"
|
||||
|
||||
- name: make the boxe be real for real
|
||||
hosts: mm:mm_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- base
|
||||
- rkhunter
|
||||
- nagios_client
|
||||
- zabbix/zabbix_agent
|
||||
- hosts
|
||||
- {role: openvpn/client, when: env != "staging" and inventory_hostname.startswith('mm-frontend')}
|
||||
- ipa/client
|
||||
- sudo
|
||||
- collectd/base
|
||||
- {role: nfs/client, when: inventory_hostname.startswith('mm-backend01'), mnt_dir: '/srv/pub', nfs_src_dir: 'fedora_ftp/fedora.redhat.com/pub', mount_stg: true}
|
||||
- {role: nfs/client, when: inventory_hostname.startswith('mm-backend01'), mnt_dir: '/srv/pub/archive', nfs_src_dir: 'fedora_ftp_archive', mount_stg: true}
|
||||
|
||||
pre_tasks:
|
||||
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
|
||||
|
||||
tasks:
|
||||
- import_tasks: "{{ tasks_path }}/motd.yml"
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
- name: Deploy the backend
|
||||
hosts: mm_backend:mm_backend_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
pre_tasks:
|
||||
- include_vars: dir=/srv/web/infra/ansible/vars/all/ ignore_files=README
|
||||
|
||||
roles:
|
||||
- mirrormanager/backend
|
||||
- geoip
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
- name: Deploy the crawler
|
||||
hosts: mm_crawler:mm_crawler_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- mirrormanager/crawler
|
||||
- {role: rsyncd,
|
||||
when: env != "staging"}
|
||||
- {role: openvpn/client, when: datacenter != "iad2"}
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
- name: Deploy the frontend (web-app)
|
||||
hosts: mm_frontend:mm_frontend_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- mirrormanager/frontend2
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
# Do this one last, since the mirrormanager user needs to exist so that it can
|
||||
# own the fedmsg certs we put in place here.
|
||||
- name: Put fedmsg stuff in place
|
||||
hosts: mm:mm_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- role: fedmsg/base
|
||||
# Set up for fedora-messaging
|
||||
- role: rabbit/user
|
||||
username: "mirrormanager{{ env_suffix }}"
|
||||
sent_topics: ^org\.fedoraproject\.{{ env_short }}\.(mirrormanager|logger)\..*
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# Expire old repo metadata from mirrormanager
|
||||
#
|
||||
# This playbook will mark all older versions of a distro version to be outdated
|
||||
# so that metalink will not serve <alternates> for old versions.
|
||||
# CAUTION: Until mirrors pick up the new content, this will mean that the master
|
||||
# mirror is the only one deemed respectable, and as such should be used very
|
||||
# sparingly!
|
||||
#
|
||||
# Before running this playbook, please make sure that:
|
||||
# 1. The new updates repo is mashed and pushed to the master mirrors
|
||||
# 2. The next UMDL run has occured to allow mirrormanager to pick up the new repo
|
||||
#
|
||||
# requires --extra-vars="product=Fedora version=23"
|
||||
#
|
||||
# Possible product: Fedora/EPEL/RHEL
|
||||
|
||||
- name: Expire old repo files
|
||||
# hosts: os_control
|
||||
# Testing for now:
|
||||
hosts: os_control_stg
|
||||
user: root
|
||||
gather_facts: False
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
- /srv/web/infra/ansible/vars/apps/mirrormanager.yml
|
||||
|
||||
vars:
|
||||
app: mirrormanager
|
||||
job_name: cmd-emergency-expire-repo-{{ product|lower|replace(" ", "-") }}-{{ version|lower|replace(" ", "-") }}
|
||||
|
||||
tasks:
|
||||
|
||||
- name: Validate parameters
|
||||
assert:
|
||||
that:
|
||||
- product is defined
|
||||
- version is defined
|
||||
fail_msg: "You need to define product and version"
|
||||
|
||||
- include_role:
|
||||
name: openshift/object
|
||||
vars:
|
||||
template: cmd-emergency-expire-repo.yml
|
||||
objectname: cmd-emergency-expire-repo.yml
|
||||
|
||||
- debug:
|
||||
msg: "You can watch the logs with 'oc -n {{ app }} logs -f job/{{ job_name }}'"
|
||||
|
||||
- name: Wait for the command to complete
|
||||
shell: oc -n {{ app }} wait --for=condition=Complete --timeout=10m job/{{ job_name }}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
- name: Do mm-backend stuff
|
||||
hosts: mm-backend01.iad2.fedoraproject.org
|
||||
user: root
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
tasks:
|
||||
- name: Copy borked proto
|
||||
copy: src=/var/lib/mirrormanager/mirrorlist_cache.proto dest=/root/mirrorlist_cache.proto-{{ ansible_date_time.date }} remote_src=yes
|
||||
|
||||
- name: Nuke borked proto
|
||||
file: path=/var/lib/mirrormanager/mirrorlist_cache.proto state=absent
|
||||
|
||||
- name: Copy old proto/files into place
|
||||
copy: src=/var/lib/mirrormanager/old/{{item}} dest=/var/lib/mirrormanager/{{item}} force=yes remote_src=yes
|
||||
with_items:
|
||||
- mirrorlist_cache.proto
|
||||
- i2_netblocks.txt
|
||||
- global_netblocks.txt
|
||||
|
||||
- name: Sync the proto
|
||||
command: /usr/local/bin/sync_pkl_to_mirrorlists.sh
|
||||
become: yes
|
||||
become_user: mirrormanager
|
||||
|
||||
- name: Do mm-proxy stuff
|
||||
hosts: mirrorlist-proxies
|
||||
user: root
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
tasks:
|
||||
- name: Restart mirrorlist1
|
||||
command: systemctl restart mirrorlist1
|
||||
|
||||
- name: Stop mirrorlist2
|
||||
command: systemctl stop mirrorlist2
|
||||
|
||||
- name: Stop mirrorlist3
|
||||
command: systemctl stop mirrorlist3
|
||||
when: datacenter == 'iad2'
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
MAILTO=root
|
||||
|
||||
## Replaced by mirrormanager in OpenShift
|
||||
#
|
||||
# # Refresh the mirrorlist cache every 15 minutes
|
||||
# # This takes about 1 minute
|
||||
# 0,15,30,45 * * * * mirrormanager /usr/local/bin/update-mirrorlist-server && /usr/local/bin/sync_cache_to_mirrorlists.sh
|
||||
#
|
||||
# # update master directory
|
||||
# # logs sent to /var/log/mirrormanager/umdl.log by default
|
||||
# # check if category Fedora EPEL needs updating every 30 minutes
|
||||
# 0,30 * * * * mirrormanager /usr/local/bin/umdl-required epel /var/log/mirrormanager/umdl-required.log
|
||||
# # check if category Fedora Linux needs updating every 30 minutes
|
||||
# 10,40 * * * * mirrormanager /usr/local/bin/umdl-required fedora /var/log/mirrormanager/umdl-required.log
|
||||
# # check if category Fedora Secondary Arches needs updating every 30 minutes
|
||||
# 20,50 * * * * mirrormanager /usr/local/bin/umdl-required fedora-secondary /var/log/mirrormanager/umdl-required.log
|
||||
# # check if category Fedora Codecs needs updating once a day
|
||||
# 15 20 * * * mirrormanager /usr/local/bin/umdl-required codecs /var/log/mirrormanager/umdl-required.log
|
||||
# # the remaining categories are updated every two hours
|
||||
# 15 */2 * * * mirrormanager /usr/local/bin/umdl-required archive /var/log/mirrormanager/umdl-required.log
|
||||
# 45 */2 * * * mirrormanager /usr/local/bin/umdl-required alt /var/log/mirrormanager/umdl-required.log
|
||||
#
|
||||
# # check if category CentOS needs updating every 15 minutes
|
||||
# 57,12,27,42 * * * * mirrormanager /usr/local/bin/umdl-required centos /var/log/mirrormanager/umdl-required.log
|
||||
#
|
||||
# # Sync netblocks list once a day
|
||||
# 30 0 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_global_netblocks /var/lib/mirrormanager/global_netblocks.txt
|
||||
# # Run the script for Internet2 netblocks very late to
|
||||
# # make sure the files are all there. The script is
|
||||
# # looking in date dependent directories and depending
|
||||
# # on the timezone of the remote server the files appear late.
|
||||
# 0 23 * * * mirrormanager cd /usr/share/mirrormanager2 && /usr/bin/mm2_get_internet2_netblocks /var/lib/mirrormanager/i2_netblocks.txt
|
||||
# 30 1 * * * mirrormanager /usr/bin/mm2_update-EC2-netblocks
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
/var/log/mirrormanager/umdl-required.log {
|
||||
missingok
|
||||
notifempty
|
||||
daily
|
||||
dateext
|
||||
rotate 15
|
||||
postrotate
|
||||
endscript
|
||||
}
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script checks for changes on the primary mirror and
|
||||
# updates the database if changes are found. For most categories
|
||||
# the script first checks if a fullfiletimelist-<category>
|
||||
# exists and runs the actual primary mirror scan if that file
|
||||
# has changed.
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
echo "Exactly two parameter needed. category and /path/to/logfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec >> $2
|
||||
exec 2>&1
|
||||
|
||||
CURDATE=`date +%s`
|
||||
SCANNER="/usr/bin/mm2_update-master-directory-list"
|
||||
|
||||
if [ "${1}" == "fedora" ]; then
|
||||
CATEGORY="Fedora Linux"
|
||||
elif [ "${1}" == "epel" ]; then
|
||||
CATEGORY="Fedora EPEL"
|
||||
SCANNER="/usr/local/bin/scan-primary-mirror"
|
||||
elif [ "${1}" == "alt" ]; then
|
||||
CATEGORY="Fedora Other"
|
||||
elif [ "${1}" == "fedora-secondary" ]; then
|
||||
CATEGORY="Fedora Secondary Arches"
|
||||
SCANNER="/usr/local/bin/scan-primary-mirror -d"
|
||||
elif [ "${1}" == "archive" ]; then
|
||||
CATEGORY="Fedora Archive"
|
||||
SCANNER="/usr/local/bin/scan-primary-mirror -d"
|
||||
elif [ "${1}" == "codecs" ]; then
|
||||
CATEGORY="Fedora Codecs"
|
||||
SCANNER="/usr/local/bin/scan-primary-mirror"
|
||||
elif [ "${1}" == "centos" ]; then
|
||||
CATEGORY="CentOS"
|
||||
SCANNER="/usr/local/bin/scan-primary-mirror -c /etc/mirrormanager/scan-primary-mirror-centos.toml -d"
|
||||
fi
|
||||
|
||||
if [ -e /var/run/mirrormanager/umdl-${1} ]; then
|
||||
. /var/run/mirrormanager/umdl-${1}
|
||||
else
|
||||
# 24 hours -> 86400 seconds
|
||||
let LASTRUN=CURDATE-86400
|
||||
fi
|
||||
|
||||
if [ "${1}" == "centos" ]; then
|
||||
CENTOS_PRIMARY="mref1-priv.iad2.centos.org"
|
||||
# check if a sync is currently in process
|
||||
CODE=$( curl -s -o /dev/null -I -w "%{http_code}" http://${CENTOS_PRIMARY}/9-stream/.sync_in_progress )
|
||||
if [ "${CODE}" -eq "200" ]; then
|
||||
echo -n "CentOS primary mirror sync in progress. Skipping scan at "
|
||||
date
|
||||
exit 0
|
||||
fi
|
||||
FFTL="http://${CENTOS_PRIMARY}/9-stream/COMPOSE_ID"
|
||||
FILEDATE=`date +%s -d"$( curl -s --head ${FFTL} | awk 'BEGIN {FS=": "}/^Last-Modified/{print $2}' )"`
|
||||
FFTL_SIGS="http://${CENTOS_PRIMARY}/SIGs/9-stream/COMPOSE_ID"
|
||||
FILEDATE_SIGS=`date +%s -d"$( curl -s --head ${FFTL_SIGS} | awk 'BEGIN {FS=": "}/^Last-Modified/{print $2}' )"`
|
||||
if [ "$FILEDATE_SIGS" -gt "$FILEDATE" ]; then
|
||||
FILEDATE=$FILEDATE_SIGS
|
||||
FFTL=$FFTL_SIGS
|
||||
fi
|
||||
elif [ "${1}" == "codecs" ]; then
|
||||
FFTL="${CATEGORY}"
|
||||
FILEDATE=${CURDATE}
|
||||
else
|
||||
FFTL="/srv/pub/${1}/fullfiletimelist-${1}"
|
||||
FILEDATE=`stat -c %Z ${FFTL} 2> /dev/null`
|
||||
|
||||
if [ "$?" -eq "1" ]; then
|
||||
echo "Error stat() of ${FFTL} failed. This should not happen."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rerun scan after 24 hours
|
||||
if [ "$LASTRUN" -gt "$FILEDATE" ] && [ "$LASTRUN" -gt $(expr $CURDATE - 86400) ]; then
|
||||
# no changes on the master mirror
|
||||
# abort
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -n "${FFTL} has changed since last run. Running umdl for ${CATEGORY} at "
|
||||
date
|
||||
|
||||
/usr/local/bin/lock-wrapper umdl-${1} "${SCANNER} --category \"${CATEGORY}\""
|
||||
|
||||
if [ "$?" -eq "0" ]; then
|
||||
# success! remember the date of this run
|
||||
echo "LASTRUN=${CURDATE}" > /var/run/mirrormanager/umdl-${1}
|
||||
echo -n "Finished umdl for ${CATEGORY} successfully at "
|
||||
date
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo -n "umdl for ${CATEGORY} returned non-zero. Something failed. Please check umdl.log. "
|
||||
date
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
LOCKFILE=/var/lock/mirrormanager/update-mirrorlist-server.lock
|
||||
CACHEDIR=/var/lib/mirrormanager
|
||||
|
||||
[ -e ${LOCKFILE} ] && kill -0 $(cat ${LOCKFILE}) && exit 2
|
||||
mkdir -p $(dirname ${LOCKFILE})
|
||||
echo $$ > ${LOCKFILE}
|
||||
trap "rm -f ${LOCKFILE}" QUIT TERM INT HUP EXIT
|
||||
|
||||
cd ${MM_DIR}
|
||||
rm -rf ${CACHEDIR}/old
|
||||
mkdir -p ${CACHEDIR}/old
|
||||
cp -ar ${CACHEDIR}/* ${CACHEDIR}/old/ 2>/dev/null
|
||||
|
||||
/usr/local/bin/generate-mirrorlist-cache $@
|
||||
|
||||
exit 0
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
---
|
||||
# Configuration for the mirrormanager backend
|
||||
|
||||
- name: install needed packages
|
||||
package: name={{ item }} state=present update_cache=yes
|
||||
with_items:
|
||||
- mirrormanager2-backend
|
||||
- mirrormanager2-statistics
|
||||
- bzip2
|
||||
- python-psycopg2
|
||||
- fedmsg
|
||||
- fedora-messaging
|
||||
- jq
|
||||
- geolite2-city
|
||||
- geolite2-country
|
||||
- cargo
|
||||
- postgresql-devel
|
||||
- git
|
||||
tags:
|
||||
- packages
|
||||
|
||||
- name: create /etc/mirrormanager
|
||||
file: path=/etc/mirrormanager state=directory
|
||||
|
||||
- name: create /var/log/mirrormanager
|
||||
file: path=/var/log/mirrormanager state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
|
||||
- name: install MM configuration file
|
||||
template: src={{ item.file }} dest={{ item.dest }}
|
||||
owner=mirrormanager group=mirrormanager mode=0600
|
||||
with_items:
|
||||
- { file: "{{ roles_path }}/mirrormanager/frontend2/templates/mirrormanager2.cfg",
|
||||
dest: /etc/mirrormanager/mirrormanager2.cfg }
|
||||
- { file: "{{ roles_path }}/mirrormanager/frontend2/templates/alembic.ini",
|
||||
dest: /etc/mirrormanager/alembic.ini }
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: install the cron job
|
||||
copy: src=backend.cron dest=/etc/cron.d/mm2_backend.cron
|
||||
tags:
|
||||
- config
|
||||
when: env != 'staging'
|
||||
|
||||
- name: install backend helper scripts
|
||||
copy: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755
|
||||
with_items:
|
||||
- umdl-required
|
||||
- update-mirrorlist-server
|
||||
tags:
|
||||
- mirrormanager/backend
|
||||
|
||||
- name: install the umdl-required logrotate file
|
||||
copy: src=mm2_umdl-required.logrotate dest=/etc/logrotate.d/mm2_umdl-required mode=644
|
||||
|
||||
- name: install backend helper scripts from template
|
||||
template: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755
|
||||
with_items:
|
||||
- sync_cache_to_mirrorlists.sh
|
||||
- handle_propagation.sh
|
||||
- create_maps.sh
|
||||
- create_statistics.sh
|
||||
tags:
|
||||
- mirrormanager/backend
|
||||
|
||||
- name: handle propagation cronjob
|
||||
cron: name="handle-propagation" minute="45" hour="*/2" user="mirrormanager"
|
||||
job="/usr/local/bin/handle_propagation.sh"
|
||||
cron_file=handle-propagation
|
||||
disabled=true
|
||||
|
||||
- name: create worldmap cronjob
|
||||
cron: name="create-worldmap" minute="50" hour="20" user="mirrormanager"
|
||||
job="/usr/local/bin/create_maps.sh"
|
||||
cron_file=create-worldmap
|
||||
disabled=true
|
||||
|
||||
- name: create mirrorlist statistics cronjob
|
||||
cron: name="create-statistics" minute="4" hour="*/2" user="mirrormanager"
|
||||
job="/usr/local/bin/create_statistics.sh"
|
||||
cron_file=create-statistics
|
||||
disabled=true
|
||||
|
||||
- name: create yesterdays mirrorlist statistics cronjob
|
||||
cron: name="yesterdays-statistics" minute="55" hour="0" user="mirrormanager"
|
||||
job="/usr/local/bin/create_statistics.sh yesterday"
|
||||
cron_file=yesterdays-statistics
|
||||
disabled=true
|
||||
|
||||
- name: setup /var/lib/mirrormanager/.ssh directory
|
||||
copy: >
|
||||
src="{{ private }}/files/mirrormanager/"
|
||||
dest="/var/lib/mirrormanager/.ssh"
|
||||
directory_mode=0700
|
||||
owner=mirrormanager
|
||||
group=mirrormanager
|
||||
mode=0700
|
||||
tags:
|
||||
- config
|
||||
when: env != 'staging'
|
||||
|
||||
- name: Create /etc/pki/fedora-messaging
|
||||
file:
|
||||
dest: /etc/pki/fedora-messaging
|
||||
mode: 0775
|
||||
owner: root
|
||||
group: root
|
||||
state: directory
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
||||
# FIXME: do we need to create a mirrormanager cert ?
|
||||
- name: Deploy the Fedora mirrormanager fedora-messaging cert
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mirrormanager2{{env_suffix}}.crt"
|
||||
dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-cert.pem
|
||||
mode: 0644
|
||||
owner: root
|
||||
group: root
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: Deploy the Fedora infra fedora-messaging key
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mirrormanager2{{env_suffix}}.key"
|
||||
dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-key.pem
|
||||
mode: 0640
|
||||
owner: root
|
||||
group: root
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
||||
# To decrease the crawl duration on the mirrors we have been
|
||||
# recommending to lower the default value of vfs_cache_pressure
|
||||
# from 100 to 10. This causes the kernel to prefer to keep dentries
|
||||
# when under memory pressure. Let's also set it on the mm backend
|
||||
# systems as umdl is also mainly looking at the metadata.
|
||||
- sysctl: name=vm.vfs_cache_pressure value=10 state=present sysctl_set=yes reload=yes
|
||||
|
||||
# MirrorManager cannot access pre-bitflip content:
|
||||
# https://fedorahosted.org/fedora-infrastructure/ticket/5289
|
||||
# Putting the mirrormanager user into the ftpsync(263) group
|
||||
# should enable the mirrormanager user to read the files.
|
||||
|
||||
- group: name=ftpsync state=present gid=263
|
||||
|
||||
# The mirrormanager user is created by the mirrormanager2-backend RPM
|
||||
# Just adding it the existing user to the ftpsync group.
|
||||
- user: name=mirrormanager groups=ftpsync append=yes
|
||||
|
||||
- name: install generate-mirrorlist-cache configuration file
|
||||
template:
|
||||
src: generate-mirrorlist-cache.cfg
|
||||
dest: /etc/mirrormanager/generate-mirrorlist-cache.cfg
|
||||
owner: mirrormanager
|
||||
group: mirrormanager
|
||||
mode: 0600
|
||||
- name: install scan-primary-mirror configuration file
|
||||
template:
|
||||
src: scan-primary-mirror.toml
|
||||
dest: /etc/mirrormanager/scan-primary-mirror.toml
|
||||
owner: mirrormanager
|
||||
group: mirrormanager
|
||||
mode: 0600
|
||||
- name: install scan-primary-mirror centos configuration file
|
||||
template:
|
||||
src: scan-primary-mirror-centos.toml
|
||||
dest: /etc/mirrormanager/scan-primary-mirror-centos.toml
|
||||
owner: mirrormanager
|
||||
group: mirrormanager
|
||||
mode: 0600
|
||||
- name: create /srv/mirrorlist-server.git
|
||||
file: path=/srv/mirrorlist-server.git state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
- name: create /srv/scan-primary-mirror.git
|
||||
file: path=/srv/scan-primary-mirror.git state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
- name: checkout mirrorlist-server
|
||||
git:
|
||||
repo: https://github.com/adrianreber/mirrorlist-server.git
|
||||
version: 3.0.6
|
||||
dest: /srv/mirrorlist-server.git
|
||||
become: yes
|
||||
become_user: mirrormanager
|
||||
register: mirrorlist_server_downloaded
|
||||
- name: checkout scan-primary-mirror
|
||||
git:
|
||||
repo: https://github.com/adrianreber/scan-primary-mirror.git
|
||||
version: 0.4.3
|
||||
dest: /srv/scan-primary-mirror.git
|
||||
become: yes
|
||||
become_user: mirrormanager
|
||||
register: scan_primary_mirror_downloaded
|
||||
- name: build generate-mirrorlist-cache
|
||||
command: "cargo build --release --bin generate-mirrorlist-cache"
|
||||
args:
|
||||
chdir: /srv/mirrorlist-server.git
|
||||
become: yes
|
||||
become_user: mirrormanager
|
||||
register: mirrorlist_server_built
|
||||
when: "mirrorlist_server_downloaded is changed"
|
||||
- name: build scan-primary-mirror
|
||||
command: "cargo build --release"
|
||||
args:
|
||||
chdir: /srv/scan-primary-mirror.git
|
||||
become: yes
|
||||
become_user: mirrormanager
|
||||
register: scan_primary_mirror_built
|
||||
when: "scan_primary_mirror_downloaded is changed"
|
||||
- name: install generate-mirrorlist-cache
|
||||
copy:
|
||||
src: /srv/mirrorlist-server.git/target/release/generate-mirrorlist-cache
|
||||
dest: /usr/local/bin/generate-mirrorlist-cache
|
||||
remote_src: yes
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
when: "mirrorlist_server_built is changed"
|
||||
- name: install scan-primary-mirror
|
||||
copy:
|
||||
src: /srv/scan-primary-mirror.git/target/release/scan-primary-mirror
|
||||
dest: /usr/local/bin/scan-primary-mirror
|
||||
remote_src: yes
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0755
|
||||
when: "scan_primary_mirror_built is changed"
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
WORLDMAP="/usr/bin/mm2_generate-worldmap"
|
||||
|
||||
FRONTENDS="{% for host in groups['mm_frontend'] %} {{ host }} {% endfor %}"
|
||||
|
||||
OUTPUT=`mktemp -d`
|
||||
|
||||
trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT
|
||||
|
||||
${WORLDMAP} --output ${OUTPUT} > /dev/null
|
||||
|
||||
for f in ${FRONTENDS}; do
|
||||
rsync -aq ${OUTPUT}/{map.png,mirrors_location.txt} ${f}:/var/www/mirrormanager-statistics/map/
|
||||
done
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
MIRRORLIST_PROXIES="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
|
||||
FRONTENDS="{% for host in groups['mm_frontend'] %} {{ host }} {% endfor %}"
|
||||
|
||||
INPUT="/var/log/mirrormanager/mirrorlist.log"
|
||||
CONTAINER1="/var/log/mirrormanager/mirrorlist1.service.log"
|
||||
CONTAINER2="/var/log/mirrormanager/mirrorlist2.service.log"
|
||||
|
||||
if [ "$1" == "yesterday" ]; then
|
||||
STATISTICS="/usr/bin/mirrorlist_statistics -o 1"
|
||||
DEST="/var/www/mirrormanager-statistics/data/`date +%Y/%m --date='yesterday'`"
|
||||
else
|
||||
STATISTICS="/usr/bin/mirrorlist_statistics"
|
||||
DEST="/var/www/mirrormanager-statistics/data/`date +%Y/%m`"
|
||||
fi
|
||||
|
||||
DATE=`date +%Y%m%d`
|
||||
OUTPUT=`mktemp -d`
|
||||
|
||||
trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}" QUIT TERM INT HUP EXIT
|
||||
|
||||
for s in ${MIRRORLIST_PROXIES}; do
|
||||
ssh $s "( cat $CONTAINER1 | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
|
||||
ssh $s "( cat $CONTAINER2 | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
|
||||
if [ "$1" == "yesterday" ]; then
|
||||
ssh $s "( xzcat $CONTAINER1-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
|
||||
ssh $s "( xzcat $CONTAINER2-${DATE}.xz | grep -v 127.0.0.1 | gzip -4 )" >> ${OUTPUT}/mirrorlist.log.gz
|
||||
fi
|
||||
done
|
||||
|
||||
${STATISTICS} -l ${OUTPUT}/mirrorlist.log.gz -d ${OUTPUT}/
|
||||
|
||||
for f in ${FRONTENDS}; do
|
||||
ssh ${f} mkdir -p ${DEST}
|
||||
rsync -aq ${OUTPUT}/{*.png,*.txt} ${f}:${DEST}
|
||||
done
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
{% if env == 'staging' %}
|
||||
DB_URL="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}"
|
||||
{% else %}
|
||||
DB_URL="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
|
||||
{% endif %}
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
URL="https://pdc.fedoraproject.org/rest_api/v1/releases/?active=True&name=Fedora"
|
||||
PROPAGATION="/usr/bin/mm2_propagation"
|
||||
SOURCE="mm-crawler01.iad2.fedoraproject.org::propagation"
|
||||
LOGBASE="/var/log/mirrormanager/propagation"
|
||||
|
||||
FRONTENDS="{% for host in groups['mm_frontend'] %} {{ host }} {% endfor %}"
|
||||
|
||||
OUTPUT=`mktemp -d`
|
||||
ACTIVE=`mktemp`
|
||||
|
||||
trap "rm -f ${OUTPUT}/*; rmdir ${OUTPUT}; rm -f ${ACTIVE}" QUIT TERM INT HUP EXIT
|
||||
|
||||
rsync -aq --delete ${SOURCE} ${LOGBASE}
|
||||
|
||||
curl -s ${URL} >> ${ACTIVE}
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "PROPAGATION: Querying the active collections failed. Exiting!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for version in `jq -r ".results[$i].version" < ${ACTIVE} | grep -v Rawhide`; do
|
||||
${PROPAGATION} --outdir ${OUTPUT} --logfiles "${LOGBASE}/f${version}*" --prefix f${version}_updates
|
||||
done
|
||||
|
||||
${PROPAGATION} --outdir ${OUTPUT} --logfiles "${LOGBASE}/development*"
|
||||
|
||||
# EPEL
|
||||
for version in 7 8 9; do
|
||||
${PROPAGATION} --outdir ${OUTPUT} --logfiles "${LOGBASE}/epel${version}*" --prefix epel${version}
|
||||
done
|
||||
|
||||
# CentOS
|
||||
for version in 9; do
|
||||
${PROPAGATION} --outdir ${OUTPUT} --logfiles "${LOGBASE}/centos${version}*" --prefix centos${version}
|
||||
done
|
||||
|
||||
for f in ${FRONTENDS}; do
|
||||
rsync -aq ${OUTPUT}/*[st]-repomd-propagation.svg ${f}:/var/www/mirrormanager-statistics/data/propagation
|
||||
rsync -aq ${OUTPUT}/epel[789]-repomd-propagation.svg ${f}:/var/www/mirrormanager-statistics/data/propagation
|
||||
rsync -aq ${OUTPUT}/centos[9]-repomd-propagation.svg ${f}:/var/www/mirrormanager-statistics/data/propagation
|
||||
done
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
common_rsync_options="--no-motd --timeout 14400"
|
||||
max_propagation_days = 0
|
||||
max_stale_days = 0
|
||||
excludes=[".*\\.snapshot", ".*/\\.~tmp~"]
|
||||
|
||||
[database]
|
||||
{% if env == 'staging' %}
|
||||
url="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}"
|
||||
{% else %}
|
||||
url="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
|
||||
{% endif %}
|
||||
|
||||
[[category]]
|
||||
name="CentOS"
|
||||
type="rsync"
|
||||
url="rsync://mref1-priv.iad2.centos.org/CentOS-Stream-Ref"
|
||||
checksum_base="http://mref1-priv.iad2.centos.org/"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/AppStream/"
|
||||
prefix="centos-appstream"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/BaseOS/"
|
||||
prefix="centos-baseos"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/CRB/"
|
||||
prefix="centos-crb"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/HighAvailability/"
|
||||
prefix="centos-highavailability"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/NFV/"
|
||||
prefix="centos-nfv"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/RT/"
|
||||
prefix="centos-rt"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^\\d+(?:-stream)?/ResilientStorage/"
|
||||
prefix="centos-resilientstorage"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^SIGs/\\d+(?:-stream)?/(?P<signame>\\S+?)/(?P<arch>\\S+?)/(?P<sigrepo>[^\\s/]+)(/?.*)"
|
||||
prefix="centos-${signame}-sig-${sigrepo}"
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
max_propagation_days = 2
|
||||
max_stale_days = 3
|
||||
excludes=[".*\\.snapshot", ".*/\\.~tmp~"]
|
||||
skip_paths_for_version=["pub/alt", "pub/archive"]
|
||||
test_paths=["/test/", "/stage/"]
|
||||
skip_repository_paths = ["Cloud", "Workstation", "Server", "drpms", "releases/test" ]
|
||||
do_not_display_paths = ["_Beta"]
|
||||
|
||||
[database]
|
||||
{% if env == 'staging' %}
|
||||
url="postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}"
|
||||
{% else %}
|
||||
url="postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}"
|
||||
{% endif %}
|
||||
|
||||
### EPEL ###
|
||||
|
||||
[[category]]
|
||||
name="Fedora EPEL"
|
||||
type="directory"
|
||||
url="/srv/pub/epel/"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/testing/[\\d]/[^M].*"
|
||||
prefix="testing-epel"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/[\\d]/[^M].*"
|
||||
prefix="epel"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/[\\d]/Modular/.*"
|
||||
prefix="epel-modular"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/testing/next/[\\d]/Everything/.*"
|
||||
prefix="epel-testing-next"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/next/[\\d]/Everything/.*"
|
||||
prefix="epel-next"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/playground/[\\d]/Everything/.*"
|
||||
prefix="epel-playground"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="pub/epel/testing/[\\d]/Modular/.*"
|
||||
prefix="testing-modular-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-modular-epel-debug-"
|
||||
to="testing-modular-debug-epel"
|
||||
|
||||
# The following repository_aliases are only necessary for EPEL
|
||||
# because it has some unusual repository names.
|
||||
|
||||
[[repository_aliases]]
|
||||
from="epel-playground-"
|
||||
to="playground-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="epel-playground-debug-"
|
||||
to="playground-debug-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="epel-playground-source-"
|
||||
to="playground-source-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-modular-epel-source-"
|
||||
to="testing-modular-source-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-epel-debug-"
|
||||
to="testing-debug-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-epel-source-"
|
||||
to="testing-source-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-epel-"
|
||||
to="testing-epel"
|
||||
|
||||
[[repository_aliases]]
|
||||
from="testing-modular-epel-"
|
||||
to="testing-modular-epel"
|
||||
|
||||
### Codecs ###
|
||||
|
||||
[[category]]
|
||||
name="Fedora Codecs"
|
||||
type="directory"
|
||||
url="/srv/codecs.fedoraproject.org/"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="codecs.fedoraproject.org/openh264/[\\d]+/.*"
|
||||
prefix="fedora-cisco-openh264"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="codecs.fedoraproject.org/openh264/epel/[\\d]+/.*"
|
||||
prefix="epel-cisco-openh264"
|
||||
|
||||
### Fedora Archive ###
|
||||
|
||||
[[category]]
|
||||
name="Fedora Archive"
|
||||
type="directory"
|
||||
url="/srv/pub/archive/"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/archive/fedora(-secondary)?(/linux)?/(core|releases|development)/([\\.\\d]+)/[^ME].*"
|
||||
prefix="fedora"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/updates/[\\.\\d]+/Everything/.*"
|
||||
prefix="updates-released"
|
||||
version_prefix="f"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/updates/[\\.\\d]+/Modular/.*"
|
||||
prefix="updates-released-modular"
|
||||
version_prefix="f"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/updates/testing/[\\.\\d]+/Everything/.*"
|
||||
prefix="updates-testing"
|
||||
version_prefix="f"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/updates/testing/[\\.\\d]+/Modular/.*"
|
||||
prefix="updates-testing-modular"
|
||||
version_prefix="f"
|
||||
|
||||
# pre modular structure
|
||||
[[repository_mapping]]
|
||||
regex="^pub/archive/fedora(-secondary)?(/linux)?/updates/testing/([\\.\\d]+)/[^ME].*"
|
||||
prefix="updates-testing"
|
||||
version_prefix="f"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/archive/fedora(-secondary)?(/linux)?/updates/([\\.\\d]+)/[^ME].*"
|
||||
prefix="updates-released"
|
||||
version_prefix="f"
|
||||
|
||||
### Fedora Secondary Arches ###
|
||||
|
||||
[[category]]
|
||||
name="Fedora Secondary Arches"
|
||||
type="directory"
|
||||
url="/srv/pub/fedora-secondary/"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/(releases|development)/([\\.\\d]+)/Everything/.*"
|
||||
prefix="fedora"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/(releases|development)/([\\.\\d]+)/Modular/.*"
|
||||
prefix="fedora-modular"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/development/rawhide/Everything/.*"
|
||||
prefix="rawhide"
|
||||
|
||||
[[repository_mapping]]
|
||||
regex="^pub/(archive/)?fedora(-secondary)?(/linux)?/development/rawhide/Modular/.*"
|
||||
prefix="rawhide-modular"
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# sync also to new mirrorlist containers on proxies
|
||||
|
||||
MIRRORLIST_PROXY="{% for host in groups['mirrorlist_proxies'] %} {{ host }} {% endfor %}"
|
||||
|
||||
for s in ${MIRRORLIST_PROXY}; do
|
||||
rsync -az --delete-delay --delay-updates --delete /usr/share/mirrormanager2/country_continent.csv /var/lib/mirrormanager/{*txt,*proto} ${s}:/srv/mirrorlist/data/mirrorlist1/ &
|
||||
done
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
URL="https://pdc.fedoraproject.org/rest_api/v1/releases/?active=True&name=Fedora"
|
||||
CRAWLER="/usr/bin/mm2_crawler"
|
||||
LOGBASE="/var/log/mirrormanager/propagation"
|
||||
|
||||
|
||||
ACTIVE=`mktemp`
|
||||
|
||||
trap "rm -f ${ACTIVE}" QUIT TERM INT HUP EXIT
|
||||
|
||||
curl -s ${URL} >> ${ACTIVE}
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "PROPAGATION: Querying the active collections failed. Exiting!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check propagation for the active branches
|
||||
for version in `jq -r '.results[] | select ( .release_type == "updates" ) | .version' < ${ACTIVE}`; do
|
||||
${CRAWLER} --category "Fedora Linux" --propagation --proppath updates/${version}/Everything/x86_64/repodata --threads 50 2>&1 | grep SHA256 > ${LOGBASE}/f${version}_updates-propagation.log.$( date +%s )
|
||||
done
|
||||
|
||||
# check propagation for the development branch
|
||||
${CRAWLER} --category "Fedora Linux" --propagation --proppath development/rawhide/Everything/x86_64/os/repodata --threads 50 2>&1 | grep SHA256 > ${LOGBASE}/development-propagation.log.$( date +%s )
|
||||
|
||||
# check propagation for EPEL
|
||||
for version in 7 8 9; do
|
||||
if [[ ${version} -lt 8 ]]; then
|
||||
${CRAWLER} --category "Fedora EPEL" --propagation --proppath ${version}/x86_64/repodata --threads 50 --timeout 1 2>&1 | grep SHA256 > ${LOGBASE}/epel${version}-propagation.log.$( date +%s )
|
||||
else
|
||||
${CRAWLER} --category "Fedora EPEL" --propagation --proppath ${version}/Everything/x86_64/repodata --threads 50 --timeout 1 2>&1 | grep SHA256 > ${LOGBASE}/epel${version}-propagation.log.$( date +%s )
|
||||
fi
|
||||
done
|
||||
|
||||
# check propagation for CentOS
|
||||
${CRAWLER} --category "CentOS" --propagation --proppath 9-stream/BaseOS/x86_64/os/repodata --threads 50 --timeout 1 2>&1 | grep SHA256 > ${LOGBASE}/centos9-propagation.log.$( date +%s )
|
||||
|
||||
# clean up log files older than 14 days
|
||||
/usr/sbin/tmpwatch --mtime 14d ${LOGBASE}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
## Replaced by mirrormanager in OpenShift
|
||||
#
|
||||
# # run the crawler for each MirrorManager category
|
||||
# # logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
|
||||
# #
|
||||
# # [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h is used to start the crawl
|
||||
# # later on the second crawler to reduce the number of parallel accesses to
|
||||
# # the database
|
||||
# #
|
||||
# # To make sure only one cron started crawler is running the previous running
|
||||
# # (cron) crawlers are being signaled to shut down. The crawler can try to
|
||||
# # gracefully shutdown if it gets the signal SIGALRM(14). After the signal we
|
||||
# # wait for 5 minutes to give the crawler a chance to shutdown. After that the
|
||||
# # crawler is killed. To make sure we only end the cron started crawler we look
|
||||
# # for the following process "/usr/bin/python /usr/bin/mm2_crawler --category=25".
|
||||
#
|
||||
# # The number of threads is based on the possible number of existing mirrors. More
|
||||
# # threads for categories with more mirrors.
|
||||
#
|
||||
# # The goal is to distribute the crawling of all categories over the whole day.
|
||||
#
|
||||
# # The timeout is 4 hours, but for each category.
|
||||
#
|
||||
# # Category: 'Fedora Linux'; twice a day, 20 threads
|
||||
# 0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Linux"; /usr/bin/mm2_crawler --category="Fedora Linux" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
#
|
||||
# # Category: 'Fedora Secondary Arches'; twice a day, 10 threads
|
||||
# 0 3,9 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Secondary Arches"; /usr/bin/mm2_crawler --category="Fedora Secondary Arches" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
#
|
||||
# # Category: 'Fedora EPEL'; four times a day, 20 threads
|
||||
# 45 */6 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 1h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora EPEL"; /usr/bin/mm2_crawler --category="Fedora EPEL" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
#
|
||||
# # Category: 'Fedora Archive'; once a day, 10 threads
|
||||
# 0 2 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Archive"; /usr/bin/mm2_crawler --category="Fedora Archive" --threads 9 --timeout-minutes 300 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
#
|
||||
# # Category: 'Fedora Other'; once a day, 10 threads
|
||||
# 0 14 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=Fedora Other"; /usr/bin/mm2_crawler --category="Fedora Other" --threads 9 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
#
|
||||
# # Category: 'CentOS'; twice a day, 20 threads
|
||||
# 0 5,11 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 6h; pkill -14 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; sleep 5m; pkill -9 -f "^/usr/bin/python2 -s /usr/bin/mm2_crawler --category=CentOS"; /usr/bin/mm2_crawler --category="CentOS" --threads 19 --timeout-minutes 240 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
import mirrormanager2.lib
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(usage=sys.argv[0] + " [options]",
|
||||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument(
|
||||
"-c", "--config",
|
||||
dest="config", default='/etc/mirrormanager/mirrormanager2.cfg',
|
||||
help="Configuration file to use")
|
||||
|
||||
parser.add_argument(
|
||||
"-f", "--fraction",
|
||||
dest="fraction", default="1:1",
|
||||
help='''Specify which part of the mirror range should be returned
|
||||
1:1 - all mirrors
|
||||
1:2 - the first half of the mirrors
|
||||
2:3 - the middle third of the mirrors''')
|
||||
|
||||
options = parser.parse_args()
|
||||
|
||||
config = dict()
|
||||
with open(options.config) as config_file:
|
||||
exec(compile(config_file.read(), options.config, 'exec'), config)
|
||||
|
||||
if ':' not in options.fraction:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
session = mirrormanager2.lib.create_session(config['DB_URL'])
|
||||
|
||||
# Get all active mirrors
|
||||
hosts = mirrormanager2.lib.get_mirrors(session, private=False,
|
||||
admin_active=True, user_active=True, site_private=False,
|
||||
site_user_active=True, site_admin_active=True)
|
||||
|
||||
# only the ids
|
||||
hosts = [ host.id for host in hosts ]
|
||||
|
||||
session.close()
|
||||
|
||||
hosts.sort()
|
||||
|
||||
total = int(options.fraction.split(':')[1])
|
||||
part = int(options.fraction.split(':')[0])
|
||||
|
||||
start = (part-1)*(len(hosts)/total)
|
||||
stop = (len(hosts)/total)*part
|
||||
|
||||
if total == part:
|
||||
print "--startid=%d" % (hosts[start])
|
||||
else:
|
||||
print "--startid=%d --stopid=%d" % (hosts[start], hosts[stop])
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#/bin/bash
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
NUMBER_OF_CRAWLERS=$1
|
||||
HOST=`hostname -s`
|
||||
START_STOP="/usr/local/bin/mm2_get-mirrors-to-crawl"
|
||||
|
||||
|
||||
for i in `seq 1 ${NUMBER_OF_CRAWLERS}`; do
|
||||
if [ "${HOST}" == "mm-crawler0${i}" ]; then
|
||||
${START_STOP} -f ${i}:${NUMBER_OF_CRAWLERS}
|
||||
fi
|
||||
done
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
---
|
||||
# Configuration for MirrorManager's crawler
|
||||
|
||||
- name: install needed packages
|
||||
package: name={{ item }} state=present update_cache=yes
|
||||
with_items:
|
||||
- mirrormanager2-crawler
|
||||
- python-psycopg2
|
||||
- fedmsg
|
||||
- python-GeoIP
|
||||
- jq
|
||||
- geolite2-city
|
||||
- geolite2-country
|
||||
tags:
|
||||
- packages
|
||||
when: ansible_hostname != "mm-crawler-dev.stg.iad2.fedoraproject.org"
|
||||
|
||||
- name: create /etc/mirrormanager
|
||||
file: path=/etc/mirrormanager state=directory
|
||||
|
||||
- name: install MM configuration file
|
||||
template: src={{ item.file }} dest={{ item.dest }}
|
||||
owner=mirrormanager group=mirrormanager mode=0600
|
||||
with_items:
|
||||
- { file: "{{ roles_path }}/mirrormanager/frontend2/templates/mirrormanager2.cfg",
|
||||
dest: /etc/mirrormanager/mirrormanager2.cfg }
|
||||
- { file: "{{ roles_path }}/mirrormanager/frontend2/templates/alembic.ini",
|
||||
dest: /etc/mirrormanager/alembic.ini }
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: install the cron job
|
||||
copy: src=crawler.cron dest=/etc/cron.d/mm2_crawler.cron
|
||||
tags:
|
||||
- config
|
||||
when: env != 'staging'
|
||||
|
||||
- name: install crawling helper scripts
|
||||
copy: src={{ item }} dest=/usr/local/bin/{{ item }} mode=0755
|
||||
with_items:
|
||||
- mm2_get-mirrors-to-crawl
|
||||
- run_crawler.sh
|
||||
- check_propagation.sh
|
||||
|
||||
# Directory used to store the result of the repomd.xml propagation test
|
||||
- name: create /var/log/mirrormanager/propagation
|
||||
file: path=/var/log/mirrormanager/propagation state=directory
|
||||
owner=mirrormanager group=mirrormanager mode=0755
|
||||
|
||||
- name: propagation cronjob
|
||||
cron: name="propagation-check" minute="27" hour="*/2" user="mirrormanager"
|
||||
job="/usr/local/bin/check_propagation.sh"
|
||||
cron_file=propagation-check
|
||||
disabled=true
|
||||
when: inventory_hostname.startswith('mm-crawler01.iad2')
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 KiB |
|
|
@ -1,70 +0,0 @@
|
|||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
<!--
|
||||
# This script is for rendering ambassadors map.
|
||||
#
|
||||
# Copyright (C) 2009, Susmit Shannigrahi, Susmit AT fedoraproject DOT org
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Openlayer.js, Openstreetmap.js, and the map data are copyrighted by their respective
|
||||
# copyright owners.
|
||||
|
||||
-->
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
|
||||
<title>Fedora Mirrors Map</title>
|
||||
<style type="text/css">
|
||||
html, body, #map { width 100%; height: 100%; }
|
||||
</style>
|
||||
<script src="OpenLayers.js" type="text/javascript"></script>
|
||||
</head>
|
||||
<body>
|
||||
<h4 id="title" style="text-align: center;">Fedora Mirrors Map</h4>
|
||||
<div id="map"></div>
|
||||
<script defer="defer" type="text/javascript">
|
||||
|
||||
var map = new OpenLayers.Map ("map");
|
||||
var Osm = new OpenLayers.Layer.OSM("Open Street Map",
|
||||
// Official OSM tileset as protocol-independent URLs
|
||||
[
|
||||
'//a.tile.openstreetmap.org/${z}/${x}/${y}.png',
|
||||
'//b.tile.openstreetmap.org/${z}/${x}/${y}.png',
|
||||
'//c.tile.openstreetmap.org/${z}/${x}/${y}.png'
|
||||
], null);
|
||||
|
||||
map.addLayer(Osm);
|
||||
|
||||
var text = new OpenLayers.Layer.Text( "Fedora Mirrors", {location: "./mirrors_location.txt"} );
|
||||
map.addLayer(text);
|
||||
var lonlat = new OpenLayers.LonLat(-15, 25).transform(
|
||||
new OpenLayers.Projection("EPSG:4326"), // transform from WGS 1984
|
||||
new OpenLayers.Projection("EPSG:900913") // to Spherical Mercator
|
||||
);
|
||||
|
||||
map.setCenter(lonlat, 3);
|
||||
|
||||
</script>
|
||||
<script src="https://apps.fedoraproject.org/fedmenu/js/jquery-1.11.2.min.js" type="text/javascript"></script>
|
||||
<script src="https://apps.fedoraproject.org/fedmenu/js/fedmenu.js" type="text/javascript"></script>
|
||||
<script type="text/javascript">
|
||||
fedmenu({
|
||||
'url': 'https://apps.fedoraproject.org/js/data.js',
|
||||
'mimeType': 'application/javascript',
|
||||
'position': 'bottom-right',
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
---
|
||||
# tasklist for setting up the mirrormanager web application
|
||||
|
||||
- name: install needed packages
|
||||
package: name={{ item }} state=present update_cache=yes
|
||||
with_items:
|
||||
- mirrormanager2
|
||||
- python-psycopg2
|
||||
- python-openid-cla
|
||||
- python-openid-teams
|
||||
- python-memcached
|
||||
- libsemanage-python
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- packages
|
||||
|
||||
- name: install configuration file
|
||||
template: src={{ item.file }} dest={{ item.dest }}
|
||||
owner=apache group=apache mode=0600
|
||||
with_items:
|
||||
- {file: mirrormanager.conf, dest: /etc/httpd/conf.d/mirrormanager.conf}
|
||||
- {file: mirrormanager2.cfg, dest: /etc/mirrormanager/mirrormanager2.cfg}
|
||||
- {file: alembic.ini, dest: /etc/mirrormanager/alembic.ini}
|
||||
- {file: mirrormanager2.wsgi, dest: /var/www/mirrormanager2.wsgi}
|
||||
notify:
|
||||
- reload httpd
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: create the database scheme
|
||||
command: /usr/bin/python2 /usr/share/mirrormanager2/mirrormanager2_createdb.py
|
||||
environment:
|
||||
MM2_CONFIG: /etc/mirrormanager/mirrormanager2.cfg
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: set sebooleans so mirrormanager can connect to its db
|
||||
seboolean: name=httpd_can_network_connect_db
|
||||
state=true
|
||||
persistent=true
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: set sebooleans so mirrormanager can send exception mails
|
||||
seboolean: name=httpd_can_sendmail
|
||||
state=true
|
||||
persistent=true
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: startup apache
|
||||
service: name=httpd enabled=yes state=started
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: Create mirrormanager user
|
||||
user: name=mirrormanager state=present home=/var/lib/mirrormanager createhome=yes shell=/bin/bash
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: create mirrormanager frontend directories
|
||||
file: path={{ item }} state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
with_items:
|
||||
- /var/log/mirrormanager
|
||||
- /var/log/mirrormanager/crawler
|
||||
- /var/www/mirrormanager-statistics
|
||||
- /var/www/mirrormanager-statistics/data
|
||||
- /var/www/mirrormanager-statistics/data/propagation
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: apply selinux type to crawler log and statistics files
|
||||
file: >
|
||||
dest={{ item }}
|
||||
setype=httpd_sys_content_t
|
||||
state=directory
|
||||
recurse=yes
|
||||
with_items:
|
||||
- /var/log/mirrormanager
|
||||
- /var/log/mirrormanager/crawler
|
||||
- /var/www/mirrormanager-statistics
|
||||
- /var/www/mirrormanager-statistics/data
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
||||
- name: install crawler log sync
|
||||
template: src=sync-crawler-logs.sh dest=/usr/local/bin/sync-crawler-logs.sh mode=0755
|
||||
tags:
|
||||
- mm2_frontend
|
||||
|
||||
- name: install the cron job
|
||||
template: src=frontend.cron dest=/etc/cron.d/mm2_frontend.cron
|
||||
tags:
|
||||
- config
|
||||
- mm2_frontend
|
||||
when: env != "staging" and inventory_hostname.startswith('mm-frontend0')
|
||||
|
||||
# mirrormanager user ssh key(s) (used to transfer statistics from backend to frontends)
|
||||
- name: add authorized_keys for mirrormanager
|
||||
authorized_key: key="{{ item }}" user=mirrormanager state=present
|
||||
with_file:
|
||||
- "{{ roles_path }}/mirrormanager/mirrorlist2/files/mm-authorized_key"
|
||||
tags:
|
||||
- mm2_frontend
|
||||
|
||||
- name: apply selinux type to mirrormanager public_key
|
||||
file: dest=/var/lib/mirrormanager/.ssh/authorized_keys setype=user_home_t state=file
|
||||
tags:
|
||||
- mm2_frontend
|
||||
- config
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = /usr/share/mirrormanager2/alembic
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
{% if env == 'staging' %}
|
||||
sqlalchemy.url = postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}
|
||||
{% else %}
|
||||
sqlalchemy.url = postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}
|
||||
{% endif %}
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
MAILTO=root
|
||||
|
||||
## Replaced by mirrormanager in OpenShift
|
||||
#
|
||||
# # sync crawler logs every hour
|
||||
# 55 * * * * mirrormanager /usr/local/bin/sync-crawler-logs.sh
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
## Replaced by mirrormanager in OpenShift
|
||||
|
||||
Redirect /mirrormanager https://mirrormanager{{ env_suffix }}.fedoraproject.org
|
||||
|
||||
# # Apache configuration file for mirrormanager2
|
||||
#
|
||||
# Alias /mirrormanager/static /usr/lib/python2.7/site-packages/mirrormanager2/static/fedora/
|
||||
# Alias /mirrormanager/crawler /var/log/mirrormanager/crawler
|
||||
# Alias /mirrormanager/data /var/www/mirrormanager-statistics/data
|
||||
# Alias /mirrormanager/map /var/www/mirrormanager-statistics/map
|
||||
#
|
||||
# WSGIDaemonProcess mirrormanager user=apache maximum-requests=100 display-name=mirrormanager processes=2 threads=4
|
||||
# WSGISocketPrefix run/wsgi
|
||||
# WSGIRestrictStdout On
|
||||
# WSGIRestrictSignal Off
|
||||
# WSGIPythonOptimize 1
|
||||
#
|
||||
# WSGIScriptAlias /mirrormanager /var/www/mirrormanager2.wsgi
|
||||
#
|
||||
# <Location />
|
||||
# WSGIProcessGroup mirrormanager
|
||||
# <IfModule mod_authz_core.c>
|
||||
# # Apache 2.4
|
||||
# Require all granted
|
||||
# </IfModule>
|
||||
# <IfModule !mod_authz_core.c>
|
||||
# # Apache 2.2
|
||||
# Order deny,allow
|
||||
# Allow from all
|
||||
# </IfModule>
|
||||
# </Location>
|
||||
#
|
||||
# <Location /mirrormanager/xmlrpc>
|
||||
# {% if mm2_checkin %}
|
||||
# Require all granted
|
||||
# {% else %}
|
||||
# Require all denied
|
||||
# {% endif %}
|
||||
# </Location>
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
'''
|
||||
MirrorManager2 sample configuration.
|
||||
'''
|
||||
|
||||
###
|
||||
# Most important configuration items
|
||||
###
|
||||
|
||||
# the number of items to display on the search pages
|
||||
# Default: ``50``.
|
||||
ITEMS_PER_PAGE = 50
|
||||
|
||||
|
||||
{% if mm2_checkin %}
|
||||
# url to the database server:
|
||||
{% if env == 'staging' %}
|
||||
DB_URL='postgresql://{{ mirrormanager_stg_checkin_db_user }}:{{ mirrormanager_stg_checkin_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}'
|
||||
{% else %}
|
||||
DB_URL='postgresql://{{ mirrormanager_checkin_db_user }}:{{ mirrormanager_checkin_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}'
|
||||
{% endif %}
|
||||
|
||||
# The checkin server does not use the secret key or password secret. Let's not leak it.
|
||||
SECRET_KEY = 'invalid'
|
||||
PASSWORD_SEED = 'invalid'
|
||||
|
||||
{% else %}
|
||||
{% if env == 'staging' %}
|
||||
# url to the database server:
|
||||
DB_URL='postgresql://{{ mirrormanager_stg_db_user }}:{{ mirrormanager_stg_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_stg_db_name }}'
|
||||
|
||||
# secret key used to generate unique csrf token
|
||||
SECRET_KEY = '{{ mirrormanager_stg_secret_key }}'
|
||||
|
||||
# Seed used to make the password harder to brute force in case of leaking
|
||||
# This should be kept really secret!
|
||||
PASSWORD_SEED = "{{ mirrormanager_stg_password_seed }}"
|
||||
{% else %}
|
||||
# url to the database server:
|
||||
DB_URL='postgresql://{{ mirrormanager_db_user }}:{{ mirrormanager_db_pass }}@{{ mirrormanager_db_host }}/{{ mirrormanager_db_name }}'
|
||||
|
||||
# secret key used to generate unique csrf token
|
||||
SECRET_KEY = '{{ mirrormanager_secret_key }}'
|
||||
|
||||
# Seed used to make the password harder to brute force in case of leaking
|
||||
# This should be kept really secret!
|
||||
PASSWORD_SEED = "{{ mirrormanager_password_seed }}"
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
|
||||
# Make browsers send session cookie only via HTTPS
|
||||
SESSION_COOKIE_SECURE=True
|
||||
|
||||
###
|
||||
# Other configuration items for the web-app
|
||||
###
|
||||
|
||||
## URLs to fedmenu resources
|
||||
{% if env == 'staging' %}
|
||||
FEDMENU_URL = 'https://apps.stg.fedoraproject.org/fedmenu'
|
||||
FEDMENU_DATA_URL = 'https://apps.stg.fedoraproject.org/js/data.js'
|
||||
{% else %}
|
||||
FEDMENU_URL = 'https://apps.fedoraproject.org/fedmenu'
|
||||
FEDMENU_DATA_URL = 'https://apps.fedoraproject.org/js/data.js'
|
||||
{% endif %}
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
# Set the time after which the session expires. Flask's default is 31 days.
|
||||
# Default: ``timedelta(hours=1)`` corresponds to 1 hour.
|
||||
PERMANENT_SESSION_LIFETIME = timedelta(hours=1)
|
||||
|
||||
# Folder containing the theme to use.
|
||||
# Default: ``fedora``.
|
||||
THEME_FOLDER = 'fedora'
|
||||
|
||||
# Which authentication method to use, defaults to `fas` can be or `local`
|
||||
# Default: ``fas``.
|
||||
MM_AUTHENTICATION = 'fas'
|
||||
|
||||
# If the authentication method is `fas`, groups in which should be the user
|
||||
# to be recognized as an admin.
|
||||
ADMIN_GROUP = ['sysadmin-main', 'sysadmin-web']
|
||||
|
||||
# Email of the admin to which send notification or error
|
||||
ADMIN_EMAIL = ['admin@fedoraproject.org', 'adrian@fedoraproject.org']
|
||||
|
||||
# Email address used in the 'From' field of the emails sent.
|
||||
# Default: ``nobody@fedoraproject.org``.
|
||||
EMAIL_FROM = 'nobody@fedoraproject.org'
|
||||
|
||||
# SMTP server to use,
|
||||
# Default: ``localhost``.
|
||||
SMTP_SERVER = 'localhost'
|
||||
|
||||
# If the SMTP server requires authentication, fill in the information here
|
||||
# SMTP_USERNAME = 'username'
|
||||
# SMTP_PASSWORD = 'password'
|
||||
|
||||
# When this is set to True, an additional menu item is shown which can
|
||||
# be used to browse the different statistics generated by
|
||||
# mirrorlist_statistics.py.
|
||||
SHOW_STATISTICS = True
|
||||
|
||||
# This is the directory the code enabled by SHOW_STATISTICS will use
|
||||
# to locate the statistics files and display them.
|
||||
STATISTICS_BASE = '/var/www/mirrormanager-statistics/data'
|
||||
|
||||
# Countries which have to be excluded.
|
||||
EMBARGOED_COUNTRIES = ['CU', 'IR', 'KP', 'SD', 'SY']
|
||||
|
||||
# When this is set to True, an additional menu item is shown which
|
||||
# displays the maps generated with mm2_generate-worldmap.
|
||||
SHOW_MAPS = True
|
||||
|
||||
# Location of the static map displayed in the map tab.
|
||||
STATIC_MAP = '/map/map.png'
|
||||
|
||||
# Location of the interactive openstreetmap based map.
|
||||
INTERACTIVE_MAP = '/map/mirrors.html'
|
||||
|
||||
# The crawler can generate propagation statistics which can be
|
||||
# converted into svg/pdf with mm2_propagation. These files
|
||||
# can be displayed next to the statistics and maps tab if desired.
|
||||
SHOW_PROPAGATION = True
|
||||
|
||||
# Where to look for the above mentioned propagation images.
|
||||
PROPAGATION_BASE = '/var/www/mirrormanager-statistics/data/propagation'
|
||||
|
||||
# Disable master rsync server ACL
|
||||
# Fedora does not use it and therefore it is set to False
|
||||
MASTER_RSYNC_ACL = False
|
||||
|
||||
# When this is set to True, the session cookie will only be returned to the
|
||||
# server via ssl (https). If you connect to the server via plain http, the
|
||||
# cookie will not be sent. This prevents sniffing of the cookie contents.
|
||||
# This may be set to False when testing your application but should always
|
||||
# be set to True in production.
|
||||
# Default: ``True``.
|
||||
MM_COOKIE_REQUIRES_HTTPS = True
|
||||
|
||||
# The name of the cookie used to store the session id.
|
||||
# Default: ``.MirrorManager``.
|
||||
MM_COOKIE_NAME = 'MirrorManager'
|
||||
|
||||
# If this variable is set (and the directory exists) the crawler
|
||||
# will create per host log files in MM_LOG_DIR/crawler/<hostid>.log
|
||||
# which can the be used in the web interface by the mirror admins.
|
||||
# Other parts besides the crawler are also using this variable to
|
||||
# decide where to store log files.
|
||||
MM_LOG_DIR = '/var/log/mirrormanager'
|
||||
|
||||
# This is used to exclude certain protocols to be entered
|
||||
# for host category URLs at all.
|
||||
# The following is the default for Fedora to exclude FTP based
|
||||
# mirrors to be added. Removing this confguration option
|
||||
# or setting it to '' removes any protocol restrictions.
|
||||
MM_PROTOCOL_REGEX = '^(?!ftp)(.*)$'
|
||||
|
||||
# The netblock size parameters define which netblock sizes can be
|
||||
# added by a site administrator. Larger networks can only be added by
|
||||
# mirrormanager admins.
|
||||
MM_IPV4_NETBLOCK_SIZE = '/16'
|
||||
MM_IPV6_NETBLOCK_SIZE = '/32'
|
||||
|
||||
# If not specified the application will rely on the root_url when sending
|
||||
# emails, otherwise it will use this URL
|
||||
# Default: ``None``.
|
||||
APPLICATION_URL = None
|
||||
|
||||
# Boolean specifying wether to check the user's IP address when retrieving
|
||||
# its session. This make things more secure (thus is on by default) but
|
||||
# under certain setup it might not work (for example is there are proxies
|
||||
# in front of the application).
|
||||
CHECK_SESSION_IP = True
|
||||
|
||||
# Specify additional rsync parameters for the crawler
|
||||
# # --timeout 14400: abort rsync crawl after 4 hours
|
||||
# # --no-human-readable: because rsync made things pretty by default in 3.1.x
|
||||
CRAWLER_RSYNC_PARAMETERS = '--no-motd --timeout 14400 --exclude=lost+found --no-human-readable'
|
||||
|
||||
# This is a list of directories which MirrorManager will ignore while guessing
|
||||
# the version and architecture from a path.
|
||||
SKIP_PATHS_FOR_VERSION = [
|
||||
'pub/alt',
|
||||
'pub/fedora/linux/releases/test',
|
||||
'pub/archive',
|
||||
'pub/fedora-secondary/development/rawhide/s390/'
|
||||
'pub/fedora-secondary/development/rawhide/Modular/ppc64/os',
|
||||
'pub/fedora-secondary/development/rawhide/Modular/ppc64/debug/tree'
|
||||
]
|
||||
|
||||
###
|
||||
# Configuration options used by the crons
|
||||
###
|
||||
|
||||
# Specify whether the crawler should send a report by email
|
||||
CRAWLER_SEND_EMAIL = False
|
||||
|
||||
# If a host fails for CRAWLER_AUTO_DISABLE times in a row
|
||||
# the host will be disable automatically (user_active)
|
||||
CRAWLER_AUTO_DISABLE = 4
|
||||
|
||||
UMDL_PREFIX = '/srv/'
|
||||
|
||||
umdl_master_directories = [
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/pub/epel/',
|
||||
'category': 'Fedora EPEL'
|
||||
},
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/pub/fedora/linux/',
|
||||
'category': 'Fedora Linux'
|
||||
},
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/pub/fedora-secondary/',
|
||||
'category': 'Fedora Secondary Arches'
|
||||
},
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/pub/archive/',
|
||||
'category': 'Fedora Archive'
|
||||
},
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/pub/alt/',
|
||||
'category': 'Fedora Other'
|
||||
},
|
||||
{
|
||||
'type': 'directory',
|
||||
'path': '/srv/codecs.fedoraproject.org/',
|
||||
'category': 'Fedora Codecs'
|
||||
},
|
||||
# {
|
||||
# 'type':'directory',
|
||||
# 'path':'../testdata/pub/fedora/linux/',
|
||||
# 'category':'Fedora Linux',
|
||||
# 'excludes':['.*/core/?.*', '.*/extras/?.*', '.*/[7-8]/?.*' ]
|
||||
# },
|
||||
# {
|
||||
# 'type':'rsync',
|
||||
# 'url':'rsync://archive.ubuntu.com/ubuntu/',
|
||||
# 'category':'Ubuntu Archive'
|
||||
# },
|
||||
# {
|
||||
# 'type':'rsync',
|
||||
# 'url':'rsync://releases.ubuntu.com/releases/',
|
||||
# 'category':'Ubuntu CD Images'
|
||||
# },
|
||||
# {
|
||||
# 'type':'rsync',
|
||||
# 'url':'rsync://ports.ubuntu.com/ubuntu-ports/',
|
||||
# 'category':'Ubuntu Ports Archive'
|
||||
# },
|
||||
# {
|
||||
# 'type':'rsync',
|
||||
# 'url':'rsync://security.ubuntu.com/ubuntu/',
|
||||
# 'category':'Ubuntu Security Archive'
|
||||
# },
|
||||
]
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#-*- coding: utf-8 -*-
|
||||
|
||||
# The three lines below are required to run on EL6 as EL6 has
|
||||
# two possible version of python-sqlalchemy and python-jinja2
|
||||
# These lines make sure the application uses the correct version.
|
||||
import __main__
|
||||
__main__.__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4']
|
||||
import pkg_resources
|
||||
|
||||
import os
|
||||
## Set the environment variable pointing to the configuration file
|
||||
os.environ['MM2_CONFIG'] = '/etc/mirrormanager/mirrormanager2.cfg'
|
||||
|
||||
## The following is only needed if you did not install mirrormanager2
|
||||
## as a python module (for example if you run it from a git clone).
|
||||
#import sys
|
||||
#sys.path.insert(0, '/path/to/mirrormanager2/')
|
||||
|
||||
|
||||
## The most import line to make the wsgi working
|
||||
from mirrormanager2.app import APP as application
|
||||
|
||||
## Turn on the debug mode to get more information in the logs about internal
|
||||
## errors
|
||||
#application.debug = True
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
CRAWLERS="{% for host in groups['mm_crawler'] %} {{ host }} {% endfor %}"
|
||||
|
||||
for i in ${CRAWLERS}; do
|
||||
rsync -aq ${i}::crawler/*log /var/log/mirrormanager/crawler/
|
||||
done
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
/var/log/mirrormanager/*.log
|
||||
{
|
||||
compress
|
||||
compresscmd /usr/bin/xz
|
||||
uncompresscmd /usr/bin/xz
|
||||
compressext .xz
|
||||
daily
|
||||
rotate 30
|
||||
missingok
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
/var/log/cron
|
||||
/var/log/maillog
|
||||
/var/log/messages
|
||||
/var/log/secure
|
||||
/var/log/spooler
|
||||
{
|
||||
sharedscripts
|
||||
postrotate
|
||||
/bin/kill -HUP `cat /var/run/syslogd.pid 2> /dev/null` 2> /dev/null || true
|
||||
endscript
|
||||
compress
|
||||
compresscmd /usr/bin/xz
|
||||
uncompresscmd /usr/bin/xz
|
||||
compressext .xz
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
.include /usr/lib/systemd/system/mirrorlist-server.service
|
||||
[Service]
|
||||
# systemd needs an empty ExecStart= to be able to overwrite ExecStart=
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/python2 /usr/share/mirrormanager2/mirrorlist_server.py -l /var/log/mirrormanager/mirrorlist.log
|
||||
|
|
@ -1 +0,0 @@
|
|||
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEA1+Mq0V8RPuCZrjsSz2y56tVR+hykwoYSodhs6ivqkuf16PHo1Aq2PR+I7xnxnJ/fqF20UEV4bmRFs9k4y/QnMOwi90O32vC3WZmfLfoY9S9dzUYXXOYy0lQZPMjmB+yOYZhhGIu2HTc4/CHKjVpCUZKHXwgkouEwsAEHQI86TrepvPwt5GrmAeZjcRp8T3maaFlRf8UFgcajF9ztgiEnBpWBEKaemtUtQ/g9cr/SwNMT3GK+M4qMVaXkuTCeiKwqxueZmTgn76aQ11sfoWYi1lVCyYt02iMDoBLaLERbwBLt3WpY/l3tQaZiPVoRPDH2EQb6v/XISsXNsqGtnc8APfhkVniURNwW/Qz8eXhpnd7GlU90iWPExvGo/Eaj2cNemgiNZH7/U7OKe3/7Li+IpPDhLCfJ8ue7Nqn+2uqhEWvzZXvamzcvEg89PBbgrdHDvJqVLnfsMDuDEo93KZ2pCfTGmjbYjbV3nnArhimLjkkb2E86489F09p65e4AoZw3HFgeW8yA3ecXtXd9FmYYkL0urT1AZFuJ/9B364h4gOqUt0oO7aW0vqhS1hW+7brIJ2DzM/vn90ONk8JVt1T3DvmgmVLxtuo1wu1PkDtekZNtJPGuGzvt5TYDjGPwZzrkYdYazLow3NMCyrhV6oNm8sxuZCLxeyZdM83Q7xty5nM= toshio@puppet1.fedora.phx.redhat.com
|
||||
|
|
@ -1 +0,0 @@
|
|||
/var/run/mirrormanager(/.*)? gen_context(system_u:object_r:httpd_var_run_t,s0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
## <summary></summary>
|
||||
Binary file not shown.
|
|
@ -1,9 +0,0 @@
|
|||
policy_module(mirrorlist2, 1.0.0)
|
||||
|
||||
require {
|
||||
type httpd_t;
|
||||
type init_t;
|
||||
class unix_stream_socket { ioctl read write create getattr setattr lock append bind connect listen accept getopt setopt shutdown connectto } ;
|
||||
}
|
||||
|
||||
allow httpd_t init_t:unix_stream_socket { ioctl read write create getattr setattr lock append bind connect listen accept getopt setopt shutdown connectto } ;
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
---
|
||||
- name: reload systemd service files
|
||||
command: systemctl daemon-reload
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
---
|
||||
# tasklist for setting up the mirrorlist app components
|
||||
# create mirrormanager user
|
||||
# create mirrormanager user
|
||||
- name: add mirrormanager user - uid {{ mirrormanager_uid }}
|
||||
user: name=mirrormanager uid={{ mirrormanager_uid }} state=present home=/home/mirrormanager createhome=yes
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
- name: make a /var/lib/mirrormanager dir for data
|
||||
file: dest=/var/lib/mirrormanager/ state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
- name: install packages for mirrorlist
|
||||
package: name={{ item }} state=present enablerepo=epel-testing
|
||||
with_items:
|
||||
- mirrormanager2-mirrorlist
|
||||
- python-IPy
|
||||
tags:
|
||||
- packages
|
||||
- mirrorlist2
|
||||
|
||||
- name: make sure mirrorlist-server is enabled (but not started yet in case we do not have pkl)
|
||||
service: name=mirrorlist-server enabled=yes
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
# install mirrorlist-server.conf apache config
|
||||
- name: mirrorlist-server apache conf
|
||||
template: src=mirrorlist-server.conf dest=/etc/httpd/conf.d/mirrorlist-server.conf
|
||||
notify:
|
||||
- restart apache
|
||||
tags:
|
||||
- config
|
||||
- mirrorlist2
|
||||
|
||||
# mirrormanager user ssh key(s)
|
||||
- name: add authorized_keys for mirrormanager
|
||||
authorized_key: key="{{ item }}" user=mirrormanager state=present
|
||||
with_file:
|
||||
- mm-authorized_key
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
|
||||
# Three tasks for handling our (two) custom selinux modules.
|
||||
- name: ensure a directory exists for our custom selinux module
|
||||
file: dest=/usr/share/mirrorlist2 state=directory
|
||||
tags:
|
||||
- mirrorlist2
|
||||
- selinux
|
||||
|
||||
- name: copy over our general mirrorlist2 selinux module
|
||||
copy: src=selinux/mirrorlist2.pp dest=/usr/share/mirrorlist2/mirrorlist2.pp
|
||||
register: ficgeneral_module
|
||||
tags:
|
||||
- mirrorlist2
|
||||
- selinux
|
||||
|
||||
#- name: check to see if its even installed yet
|
||||
# shell: semodule -l | grep mirrorlist2 | wc -l
|
||||
# register: ficgeneral_grep
|
||||
# check_mode: no
|
||||
# changed_when: "'0' in ficgeneral_grep.stdout"
|
||||
# tags:
|
||||
# - mirrorlist2
|
||||
# - selinux
|
||||
|
||||
#- name: install our general mirrorlist2 selinux module
|
||||
# command: semodule -i /usr/share/mirrorlist2/mirrorlist2.pp
|
||||
# when: ficgeneral_module is changed or ficgeneral_grep is changed
|
||||
# tags:
|
||||
# - mirrorlist2
|
||||
# - selinux
|
||||
|
||||
|
||||
- name: make a /var/log/mirrormanager dir for logs
|
||||
file: dest=/var/log/mirrormanager/ state=directory owner=mirrormanager group=mirrormanager mode=0755
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
- name: copy systemd service file in for mirrorlist-server
|
||||
copy: src=mirrorlist-server.service dest=/etc/systemd/system/mirrorlist-server.service
|
||||
notify:
|
||||
- reload systemd service files
|
||||
- restart mirrorlist-server
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
- name: setup logrotate log for mirrorlists
|
||||
copy: src=logrotate-syslog dest=/etc/logrotate.d/syslog
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
- name: setup logrotate log for mirrormanager log files
|
||||
copy: src=logrotate-mirrormanager dest=/etc/logrotate.d/mirrormanager
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
# Copy the mirrorlist log file every hour to be ready to be processed
|
||||
- name: mirrorlist copy cron
|
||||
cron: name="copy-mirrorlist" minute="50" hour="*/2" user="mirrormanager"
|
||||
job="cp /var/log/mirrormanager/mirrorlist.log /var/log/mirrormanager/mirrorlist.log.`date +\%Y-\%m-\%d`"
|
||||
cron_file=copy-mirrorlist
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
# At the start of the day the mirrorlist log file is moved
|
||||
- name: mirrorlist move cron
|
||||
cron: name="move-mirrorlist" minute="1" hour="0" user="mirrormanager"
|
||||
job="mv /var/log/mirrormanager/mirrorlist.log /var/log/mirrormanager/mirrorlist.log.`date +\%Y-\%m-\%d --date='yesterday'`"
|
||||
cron_file=move-mirrorlist
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
||||
# Cleanup old mirrorlist logfile
|
||||
- name: mirrorlist clean cron
|
||||
cron: name="clean-mirrorlist" minute="13" hour="13" user="mirrormanager"
|
||||
job="/usr/sbin/tmpwatch --mtime 7d /var/log/mirrormanager"
|
||||
cron_file=clean-mirrorlist
|
||||
tags:
|
||||
- mirrorlist2
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
#
|
||||
# mirrormanager2 moves the publiclist web page to the frontend server
|
||||
#
|
||||
{% if env == "staging" %}
|
||||
Redirect /publiclist https://admin.stg.fedoraproject.org/mirrormanager
|
||||
RewriteEngine On
|
||||
RewriteRule ^/publiclist/(.+[^/])/?$ https://admin.stg.fedoraproject.org/mirrormanager/mirrors/$1 [R,L]
|
||||
{% else %}
|
||||
Redirect /publiclist https://admin.fedoraproject.org/mirrormanager
|
||||
Alias /static /var/lib/mirrormanager/mirrorlists/static/
|
||||
RewriteEngine On
|
||||
RewriteRule ^/publiclist/(.+[^/])/?$ https://admin.fedoraproject.org/mirrormanager/mirrors/$1 [R,L]
|
||||
{% endif %}
|
||||
|
||||
<Directory /var/lib/mirrormanager/mirrorlists>
|
||||
Options Indexes FollowSymLinks
|
||||
</Directory>
|
||||
|
||||
WSGIDaemonProcess mirrorlist user=apache processes={{ mirrorlist_procs }} threads=1 display-name=mirrorlist maximum-requests=1000
|
||||
|
||||
WSGIScriptAlias /metalink /usr/share/mirrormanager2/mirrorlist_client.wsgi
|
||||
WSGIScriptAlias /mirrorlist /usr/share/mirrormanager2/mirrorlist_client.wsgi
|
||||
|
||||
# Set this if you do not have a Reverse Proxy (HTTP Accelerator) that
|
||||
# is in front of your application server running this code.
|
||||
# SetEnv mirrorlist_client.noreverseproxy 1
|
||||
|
||||
ServerLimit 900
|
||||
MaxRequestWorkers 900
|
||||
|
||||
<Location /mirrorlist>
|
||||
WSGIProcessGroup mirrorlist
|
||||
<IfModule mod_authz_core.c>
|
||||
# Apache 2.4
|
||||
Require all granted
|
||||
</IfModule>
|
||||
<IfModule !mod_authz_core.c>
|
||||
# Apache 2.2
|
||||
Order deny,allow
|
||||
Allow from all
|
||||
</IfModule>
|
||||
</Location>
|
||||
|
||||
<Location /metalink>
|
||||
WSGIProcessGroup mirrorlist
|
||||
<IfModule mod_authz_core.c>
|
||||
# Apache 2.4
|
||||
Require all granted
|
||||
</IfModule>
|
||||
<IfModule !mod_authz_core.c>
|
||||
# Apache 2.2
|
||||
Order deny,allow
|
||||
Allow from all
|
||||
</IfModule>
|
||||
</Location>
|
||||
|
||||
<Location /static>
|
||||
<IfModule mod_authz_core.c>
|
||||
# Apache 2.4
|
||||
Require all granted
|
||||
</IfModule>
|
||||
<IfModule !mod_authz_core.c>
|
||||
# Apache 2.2
|
||||
Order deny,allow
|
||||
Allow from all
|
||||
</IfModule>
|
||||
</Location>
|
||||
|
||||
#
|
||||
# Listen on 443 here
|
||||
#
|
||||
Listen 443
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
mirrormanager_uid: 441
|
||||
mirrormanager_gid: 441
|
||||
mirrors_gid: 263
|
||||
mirrors2_gid: 529
|
||||
Loading…
Add table
Add a link
Reference in a new issue