Clean up non openshift anitya in favor of openshift version.

This commit is contained in:
Kevin Fenzi 2018-10-06 00:11:10 +00:00
parent 8a59052946
commit b2ff9078f2
33 changed files with 4 additions and 1317 deletions

View file

@ -1,61 +0,0 @@
---
# Define resources for this group of hosts here.
lvm_size: 20000
mem_size: 8192
num_cpus: 2
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
custom_rules: [
# Need for rsync from log01 for logs.
'-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT',
'-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT',
]
# No other ports open. no web service running here.
#tcp_ports: []
fas_client_groups: sysadmin-noc,sysadmin-veteran
freezes: false
# Don't use testing repos in production
testing: False
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: shell
owner: root
group: sysadmin
can_send:
- logger.log
- service: anitya
owner: root
group: fedmsg
can_send:
- anitya.project.version.update
fedmsg_prefix: org.release-monitoring
fedmsg_env: prod
# For the MOTD
csi_security_category: Low
csi_primary_contact: Fedora admins - admin@fedoraproject.org
csi_purpose: Run the 'anitya' backend cronjobs and database server
csi_relationship: |
There are a few things running here:
- A number of cronjobs that scrape upstreams for new releases
- A postgres database server to be used by those crons and by
anitya-frontend01
- This host relies on:
- The fedmsg-relay daemon running on anitya-frontend01.
- Lots of external third-party services. The cronjobs make all kinds of
requests out to the Internet that can fail in various ways.
- Things that rely on this host:
- The webapps running on anitya-frontend01 relies on the postgres db
server running on this node.

View file

@ -1,81 +0,0 @@
---
# Define resources for this group of hosts here.
lvm_size: 20000
mem_size: 2048
num_cpus: 2
# for systems that do not match the above - specify the same parameter in
# the host_vars/$hostname file
# 9940 is for the anitya public relay
tcp_ports: [ 80, 443, 9940 ]
custom_rules: [
# Need for rsync from log01 for logs.
'-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT',
'-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT',
# Need so that anitya-backend can talk fedmsg to our relay
'-A INPUT -p tcp -m tcp -s 140.211.169.230 --dport 9941 -j ACCEPT',
]
fas_client_groups: sysadmin-noc,sysadmin-web,sysadmin-veteran
# Don't use testing repos in production
testing: False
freezes: false
vpn: true
# These are consumed by a task in roles/fedmsg/base/main.yml
fedmsg_certs:
- service: shell
owner: root
group: sysadmin
can_send:
- logger.log
- service: anitya
owner: root
group: apache
can_send:
- anitya.distro.add
- anitya.distro.edit
- anitya.distro.remove
- anitya.project.add
- anitya.project.add.tried
- anitya.project.edit
- anitya.project.flag
- anitya.project.flag.set
- anitya.project.map.new
- anitya.project.map.remove
- anitya.project.map.update
- anitya.project.remove
- anitya.project.version.remove
- anitya.project.version.update
fedmsg_prefix: org.release-monitoring
fedmsg_env: prod
# For the MOTD
csi_security_category: Low
csi_primary_contact: Fedora admins - admin@fedoraproject.org
csi_purpose: Run the 'anitya' mod_wsgi app for release-monitoring.org
csi_relationship: |
There are a few things running here:
- The apache/mod_wsgi app for release-monitoring.org
- A fedmsg-relay instance for anitya's local fedmsg bus
- This host relies on:
- A postgres db server running on anitya-backend01
- Lots of external third-party services. The anitya webapp can scrape
pypi, rubygems.org, sourceforge and many others on command.
- Things that rely on this host:
- The Fedora Infrastructure bus subscribes to the anitya bus published
here by the local fedmsg-relay daemon at
tcp://release-monitoring.org:9940
- the-new-hotness is a fedmsg-hub plugin running in FI on hotness01. It
listens for anitya messages from here and performs actions on koji and
bugzilla.
- anitya-backend01 expects to publish fedmsg messages via
anitya-frontend01's fedmsg-relay daemon. Access should be restricted by
firewall.

View file

@ -1,26 +0,0 @@
---
nm: 255.255.255.128
gw: 140.211.169.193
dns: 8.8.8.8
volgroup: /dev/vg_guests
eth0_ip: 140.211.169.230
eth0_nm: 255.255.255.128
fedmsg_fqdn: anitya-backend01.vpn.fedoraproject.org
ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext
ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL7-x86_64/
postfix_group: vpn
vmhost: osuosl03.fedoraproject.org
datacenter: osuosl
#
# Only allow postgresql access from the frontend node.
#
custom_rules: [ '-A INPUT -p tcp -m tcp -s 140.211.169.229 --dport 5432 -j ACCEPT' ]
sudoers: "{{ private }}/files/sudo/anitya-backend01-sudoers"
db_backup_dir: ['/backups']

View file

@ -1,23 +0,0 @@
---
nm: 255.255.255.128
gw: 140.211.169.193
dns: 8.8.8.8
ks_url: http://infrastructure.fedoraproject.org/repo/rhel/ks/kvm-rhel-7-ext
ks_repo: http://infrastructure.fedoraproject.org/repo/rhel/RHEL7-x86_64/
volgroup: /dev/vg_guests
eth0_ip: 140.211.169.229
eth0_nm: 255.255.255.128
has_ipv6: yes
eth0_ipv6: "2605:bc80:3010:600:dead:beef:cafe:fed2"
eth0_ipv6_gw: "2605:bc80:3010:600::1"
fedmsg_fqdn: anitya-frontend01.vpn.fedoraproject.org
postfix_group: vpn
vmhost: osuosl03.fedoraproject.org
datacenter: osuosl
sudoers: "{{ private }}/files/sudo/anitya-frontend01-sudoers"

View file

@ -13,6 +13,7 @@ ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/
# This is a generic list, monitored by collectd
databases:
- askfedora
- anitya
- autocloud
- blockerbugs
- bodhi
@ -36,6 +37,7 @@ databases:
# This is a more strict list of databases to backup every day
dbs_to_backup:
- askfedora
- anitya
- autocloud
- blockerbugs
- bodhi

View file

@ -655,16 +655,6 @@ pkgs02.phx2.fedoraproject.org
[pkgs-stg]
pkgs01.stg.phx2.fedoraproject.org
[anitya-backend]
anitya-backend01.fedoraproject.org
[anitya-frontend]
anitya-frontend01.fedoraproject.org
[anitya:children]
anitya-backend
anitya-frontend
[loopabull]
loopabull01.phx2.fedoraproject.org
@ -1018,7 +1008,6 @@ undercloud02.cloud.fedoraproject.org
[fedmsg-relays:children]
busgateway
anitya-frontend
[fedmsg-relays-stg:children]
busgateway-stg

View file

@ -12,7 +12,6 @@
#
# group playbooks
#
- import_playbook: /srv/web/infra/ansible/playbooks/groups/anitya.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/arm-qa.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/autocloud-backend.yml
- import_playbook: /srv/web/infra/ansible/playbooks/groups/autocloud-web.yml
@ -114,12 +113,12 @@
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/bodhi.yml
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/greenwave.yml
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/librariesio2fedmsg.yml
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/release-monitoring.yml
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/waiverdb.yml
# These need work to finish and complete and are all stg currently.
#- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/koschei.yml
#- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/modernpaste.yml
#- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/rats.yml
- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/release-monitoring.yml
#- import_playbook: /srv/web/infra/ansible/playbooks/openshift-apps/transtats.yml
#

View file

@ -1,79 +0,0 @@
# create a new sks keyserver
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=anitya"
- name: make the boxen be real for real
hosts: anitya
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- base
- rkhunter
- nagios_client
- hosts
- fas_client
- sudo
- collectd/base
- openvpn/client
tasks:
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
- import_tasks: "{{ tasks_path }}/2fa_client.yml"
- import_tasks: "{{ tasks_path }}/motd.yml"
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: set up the frontend bits
hosts: anitya-frontend
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- apache
- anitya/fedmsg
- anitya/frontend
- role: collectd/fedmsg-service
process: fedmsg-relay
tasks:
- name: install fedmsg-relay
package: name=fedmsg-relay state=present
- name: and start it
service: name=fedmsg-relay state=started
tags:
- anitya
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: set up the backend bits
hosts: anitya-backend
user: root
gather_facts: True
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
roles:
- anitya/fedmsg
- anitya/backend
tags:
- anitya
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"

View file

@ -1,72 +0,0 @@
- name: push packages out
hosts: anitya-frontend:anitya-backend
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
tasks:
- name: clean all metadata {%if testing%}(with infrastructure-testing on){%endif%}
command: yum clean all {%if testing%} --enablerepo=infrastructure-tags-stg {%endif%}
check_mode: no
- name: yum update anitya packages from main repo
yum: name="anitya" state=latest
when: not testing
- name: yum update anitya packages from testing repo
yum: name="anitya" state=latest enablerepo=infrastructure-tags-stg
when: testing
- name: verify the frontend
hosts: anitya-frontend
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
pre_tasks:
- name: tell nagios to shush w.r.t. the frontend
nagios: action=downtime minutes=15 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true
roles:
- anitya/frontend
- anitya/fedmsg
post_tasks:
- name: tell nagios to unshush w.r.t. the frontend
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true
- service: name="httpd" state=restarted
- name: verify the backend and then upgrade the db
hosts: anitya-backend
user: root
vars_files:
- /srv/web/infra/ansible/vars/global.yml
- "/srv/private/ansible/vars.yml"
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
pre_tasks:
- name: tell nagios to shush w.r.t. the backend
nagios: action=downtime minutes=15 service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true
roles:
- anitya/backend
- anitya/fedmsg
tasks:
- name: Upgrade the database
command: /usr/bin/alembic -c /etc/anitya/alembic.ini upgrade head
args:
chdir: /usr/share/anitya/
when: inventory_hostname.startswith('anitya-backend')
post_tasks:
- name: tell nagios to unshush w.r.t. the backend
nagios: action=unsilence service=host host={{ inventory_hostname_short }}{{ env_suffix }}
delegate_to: noc01.phx2.fedoraproject.org
ignore_errors: true

View file

@ -1,3 +0,0 @@
# Checks bi-daily for new versions
#
10 */12 * * * root time ANITYA_WEB_CONFIG=/etc/anitya/anitya.cfg /usr/local/bin/lock-wrapper anitya /usr/share/anitya/anitya_cron.py

View file

@ -1,10 +0,0 @@
#!/bin/bash
# Backup a database *locally* to /backups/.
DB=$1
# Make our latest backup
/usr/bin/pg_dump -C $DB | /usr/bin/xz > /backups/$DB-$(date +%F).dump.xz
# Also, delete the backup from a few days ago.
rm -f /backups/$DB-$(date --date="3 days ago" +%F).dump.xz

View file

@ -1,80 +0,0 @@
# PostgreSQL Client Authentication Configuration File
# ===================================================
#
# Refer to the PostgreSQL Administrator's Guide, chapter "Client
# Authentication" for a complete description. A short synopsis
# follows.
#
# This file controls: which hosts are allowed to connect, how clients
# are authenticated, which PostgreSQL user names they can use, which
# databases they can access. Records take one of these forms:
#
# local DATABASE USER METHOD [OPTION]
# host DATABASE USER CIDR-ADDRESS METHOD [OPTION]
# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTION]
#
# (The uppercase items must be replaced by actual values.)
#
# The first field is the connection type: "local" is a Unix-domain socket,
# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an
# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket.
#
# DATABASE can be "all", "sameuser", "samerole", a database name, or
# a comma-separated list thereof.
#
# USER can be "all", a user name, a group name prefixed with "+", or
# a comma-separated list thereof. In both the DATABASE and USER fields
# you can also write a file name prefixed with "@" to include names from
# a separate file.
#
# CIDR-ADDRESS specifies the set of hosts the record matches.
# It is made up of an IP address and a CIDR mask that is an integer
# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies
# the number of significant bits in the mask. Alternatively, you can write
# an IP address and netmask in separate columns to specify the set of hosts.
#
# METHOD can be "trust", "reject", "md5", "crypt", "password",
# "krb5", "ident", or "pam". Note that "password" sends passwords
# in clear text; "md5" is preferred since it sends encrypted passwords.
#
# OPTION is the ident map or the name of the PAM service, depending on METHOD.
#
# Database and user names containing spaces, commas, quotes and other special
# characters must be quoted. Quoting one of the keywords "all", "sameuser" or
# "samerole" makes the name lose its special character, and just match a
# database or username with that name.
#
# This file is read on server startup and when the postmaster receives
# a SIGHUP signal. If you edit the file on a running system, you have
# to SIGHUP the postmaster for the changes to take effect. You can use
# "pg_ctl reload" to do that.
# Put your actual configuration here
# ----------------------------------
#
# If you want to allow non-local connections, you need to add more
# "host" records. In that case you will also need to make PostgreSQL listen
# on a non-local interface via the listen_addresses configuration parameter,
# or via the -i or -h command line switches.
#
#@authcomment@
# TYPE DATABASE USER CIDR-ADDRESS METHOD
#@remove-line-for-nolocal@# "local" is for Unix domain socket connections only
#@remove-line-for-nolocal@local all all @authmethod@
# IPv4 local connections:
#host all all 127.0.0.1/32 @authmethod@
# IPv6 local connections:
#host all all ::1/128 @authmethod@
local all all ident
host koji koji 10.5.128.166 255.255.255.255 md5
host all all 0.0.0.0 0.0.0.0 md5
# Note, I can't think of a reason to make this more restrictive than ipv4 but
# only fakefas needs it so far
host all all ::1/128 md5

View file

@ -1,3 +0,0 @@
---
- name: restart postgresql
service: name=postgresql state=restarted

View file

@ -1,121 +0,0 @@
---
# Configuration for the anitya webapp
- name: install needed packages
package: name={{ item }} state=present update_cache=yes
with_items:
- anitya
- python-psycopg2
- libsemanage-python
- postgresql-server
tags:
- packages
- anitya_backend
- name: Initialize postgres if necessary
command: /usr/bin/postgresql-setup initdb
creates=/var/lib/pgsql/data
notify:
- restart postgresql
tags:
- anitya_backend
- name: Set postgresql-server to run on boot
service: name=postgresql enabled=yes
ignore_errors: true
notify:
- restart postgresql
tags:
- service
- anitya_backend
- name: Ensure postgres has a place to backup to
file: dest=/backups state=directory owner=postgres
tags:
- anitya_backend
- name: Copy over backup scriplet
copy: src=backup-database dest=/usr/local/bin/backup-database mode=0755
tags:
- anitya_backend
- name: Set up some cronjobs to backup databases as configured
template: >
src=cron-backup-database
dest=/etc/cron.d/cron-backup-database-{{ item }}
with_items:
- anitya
tags:
- cron
- postgresql
- name: Add our postgres config file.
copy: >
src={{ item }}
dest=/var/lib/pgsql/data/{{ item }}
owner=postgres
with_items:
- pg_hba.conf
notify:
- restart postgresql
tags:
- config
- postgresql
- name: Let postgresql listen to '*'
command: sed -i -e "s|#listen_addresses = 'localhost'|listen_addresses = '*'|" /var/lib/pgsql/data/postgresql.conf
notify:
- restart postgresql
tags:
- anitya_backend
- name: Set up some cronjobs to backup databases as configured
template: >
src=cron-backup-database
dest=/etc/cron.d/cron-backup-database-{{ item }}
with_items:
- "{{ dbs_to_backup }}"
when: dbs_to_backup != []
tags:
- cron
- anitya_backend
- name: copy sundry anitya configuration
template: src={{ item.file }}
dest={{ item.location }}/{{ item.dest }}
owner=root group=root mode=0600
with_items:
- { file: anitya_admin.cfg, location: /etc/anitya, dest: anitya.cfg }
- { file: alembic.ini, location: /etc/anitya, dest: alembic.ini }
changed_when: "1 != 1"
tags:
- config
- anitya_backend
- name: create the database scheme
command: /usr/bin/python2 /usr/share/anitya/anitya_createdb.py
environment:
ANITYA_WEB_CONFIG: /etc/anitya/anitya.cfg
tags:
- anitya_backend
- name: Install the configuration file of anitya
template: src={{ item.file }}
dest={{ item.location }}/{{ item.file }}
owner=root group=root mode=0600
with_items:
- { file: anitya.cfg, location: /etc/anitya }
tags:
- config
- anitya_backend
- name: Install the cron job
copy: src={{ item.file }}
dest={{ item.location }}/{{ item.file }}
with_items:
- { file: 'anitya.cron', location: /etc/cron.d }
tags:
- cron
- config
- anitya_backend

View file

@ -1,59 +0,0 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = /usr/share/anitya/alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
#sqlalchemy.url = driver://user:pass@localhost/dbname
sqlalchemy.url = postgresql://{{ anitya_db_admin_user }}:{{ anitya_db_admin_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View file

@ -1,29 +0,0 @@
# Beware that the quotes around the values are mandatory
### Secret key for the Flask application
SECRET_KEY='{{ anitya_secret_key }}'
### url to the database server:
#DB_URL=mysql://user:pass@host/db_name
#DB_URL=postgres://user:pass@host/db_name
DB_URL='postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}'
# List of admins based on their openid
CNUCNU_WEB_ADMINS = [
'http://ralph.id.fedoraproject.org/',
'http://pingou.id.fedoraproject.org/',
]
# Fedora OpenID endpoint
{% if env == 'staging' %}
CNUCNU_WEB_FEDORA_OPENID = 'https://id.stg.fedoraproject.org'
{% else %}
CNUCNU_WEB_FEDORA_OPENID = 'https://id.fedoraproject.org'
{% endif %}
# This is required to fix login
PREFERRED_URL_SCHEME='https'
# Make browsers send session cookie only via HTTPS
SESSION_COOKIE_SECURE = True

View file

@ -1,29 +0,0 @@
# Beware that the quotes around the values are mandatory
### Secret key for the Flask application
SECRET_KEY='{{ anitya_secret_key }}'
### url to the database server:
#DB_URL=mysql://user:pass@host/db_name
#DB_URL=postgres://user:pass@host/db_name
DB_URL='postgresql://{{ anitya_db_admin_user }}:{{ anitya_db_admin_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}'
# List of admins based on their openid
CNUCNU_WEB_ADMINS = [
'http://ralph.id.fedoraproject.org/',
'http://pingou.id.fedoraproject.org/',
]
# Fedora OpenID endpoint
{% if env == 'staging' %}
CNUCNU_WEB_FEDORA_OPENID = 'https://id.stg.fedoraproject.org'
{% else %}
CNUCNU_WEB_FEDORA_OPENID = 'https://id.fedoraproject.org'
{% endif %}
# This is required to fix login
PREFERRED_URL_SCHEME='https'
# Make browsers send session cookie only via HTTPS
SESSION_COOKIE_SECURE = True

View file

@ -1 +0,0 @@
0 0 * * * postgres /usr/local/bin/backup-database {{ item }}

View file

@ -1,21 +0,0 @@
module fedmsg 1.1;
require {
type anon_inodefs_t;
type httpd_t;
class file write;
}
require {
type ptmx_t;
type httpd_t;
class chr_file getattr;
}
#============= httpd_t ==============
# For basic port binding
allow httpd_t anon_inodefs_t:file write;
# So that psutil can work from /etc/fedmsg.d/logging.py
allow httpd_t ptmx_t:chr_file getattr;

View file

@ -1,133 +0,0 @@
---
# tasklist for setting up fedmsg
# This is the base set of files needed for fedmsg
- name: install needed packages
package: name={{ item }} state=present
with_items:
- fedmsg
- libsemanage-python
- python-psutil
- policycoreutils-python # This is in the kickstart now. Here for old hosts.
tags:
- packages
- anitya/fedmsg
# We use setgid here so that the monitoring sockets created by fedmsg services
# are accessible to the nrpe group.
- name: create a /var/run/fedmsg dir with setgid for monitoring.
file: >
dest=/var/run/fedmsg
mode=2775
owner=fedmsg
group=nrpe
state=directory
tags:
- anitya/fedmsg
- name: setup /etc/fedmsg.d directory
file: path=/etc/fedmsg.d owner=root group=root mode=0755 state=directory
tags:
- anitya/fedmsg
- config
# Any files that change need to restart any services that depend on them. A
# trick here is that some hosts have an httpd that uses fedmsg, while others do
# not. Some hosts have a fedmsg-hub that uses this config, while others do not.
# Our handlers in handlers/restart_services.yml are smart enough to
# *conditionally* restart these services, only if they are installed on the
# system.
- name: setup basic /etc/fedmsg.d/ contents
template: >
src="{{ item }}.j2"
dest="/etc/fedmsg.d/{{ item }}"
owner=root
group=root
mode=644
with_items:
- ssl.py
- relay.py
- logging.py
- base.py
tags:
- config
- fedmsgdconfig
- anitya/fedmsg
notify:
- reload httpd
- restart fedmsg-relay
- name: Remove unwanted files
file: dest=/etc/fedmsg.d/{{item}} state=absent
with_items:
- endpoints.py
tags:
- config
- fedmsgdconfig
- anitya/fedmsg
notify:
- reload httpd
- restart fedmsg-relay
- name: setup /etc/pki/fedmsg directory
file: path=/etc/pki/fedmsg owner=root group=root mode=0755 state=directory
tags:
- config
- anitya/fedmsg
- name: install fedmsg ca.cert
copy: >
src="{{ private }}/files/fedmsg-certs/keys/ca.crt"
dest=/etc/pki/fedmsg/ca.crt
owner=root
group=root
mode=0644
tags:
- config
- anitya/fedmsg
- name: fedmsg certs
copy: >
src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(inventory_hostname)}}.crt"
dest=/etc/pki/fedmsg/
mode=644
owner={{item['owner']}}
group={{item['group']}}
with_items:
- "{{ fedmsg_certs }}"
when: fedmsg_certs != []
tags:
- config
- anitya/fedmsg
- name: fedmsg keys
copy: >
src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(inventory_hostname)}}.key"
dest=/etc/pki/fedmsg/
mode=0640
owner={{item['owner']}}
group={{item['group']}}
with_items:
- "{{ fedmsg_certs }}"
when: fedmsg_certs != []
tags:
- config
- anitya/fedmsg
# Three tasks for handling our custom selinux module
- name: ensure a directory exists for our custom selinux module
file: dest=/usr/local/share/fedmsg state=directory
tags:
- anitya/fedmsg
- name: copy over our custom selinux module
copy: src=selinux/fedmsg.pp dest=/usr/local/share/fedmsg/fedmsg.pp
register: selinux_module
tags:
- anitya/fedmsg
- name: install our custom selinux module
command: semodule -i /usr/local/share/fedmsg/fedmsg.pp
when: selinux_module is changed
tags:
- anitya/fedmsg

View file

@ -1,63 +0,0 @@
config = dict(
topic_prefix="{{ fedmsg_prefix }}",
environment="{{ fedmsg_env }}",
# This used to be set to 1 for safety, but it turns out it was
# excessive. It is the number of seconds that fedmsg should sleep
# after it has initialized, but before it begins to try and send any
# messages. If set to a non-zero value, this will slow down one-off
# fedmsg scripts like the git post-receive hook and pkgdb2branch.
# If we are experiencing message-loss problems, one of the first things
# to try should be to turn this number up to a non-zero value. '1' should
# be more than sufficient.
post_init_sleep=0.4,
# This is the number of milliseconds to wait before timing out on
# connections.. notably to the fedmsg-relay in the event that it has
# crashed.
zmq_linger=2000,
# Default is 0
high_water_mark=0,
io_threads=1,
# We almost always want the fedmsg-hub to be sending messages with zmq as
# opposed to amqp or stomp. The only exception will be the bugzilla
# amqp<->zmq bridge service.
zmq_enabled=True,
# When subscribing to messages, we want to allow splats ('*') so we tell the
# hub to not be strict when comparing messages topics to subscription
# topics.
zmq_strict=False,
# See the following
# - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html
# - http://api.zeromq.org/3-2:zmq-setsockopt
zmq_tcp_keepalive=1,
zmq_tcp_keepalive_cnt=3,
zmq_tcp_keepalive_idle=60,
zmq_tcp_keepalive_intvl=5,
)
# This option adds an IPC socket by which we can monitor hub health.
try:
import os
import psutil
pid = os.getpid()
proc = [p for p in psutil.process_iter() if p.pid == pid][0]
# proc.name is a method on modern versions of psutil.
name = proc.name
if callable(name):
name = name()
config['moksha.monitoring.socket'] = \
'ipc:///var/run/fedmsg/monitoring-%s.socket' % name
config['moksha.monitoring.socket.mode'] = '770'
except (OSError, ImportError):
# We run into issues when trying to import psutil from mod_wsgi on rhel7
# but this feature is of no concern in that context, so just fail quietly.
# https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837
pass

View file

@ -1,162 +0,0 @@
# Setup fedmsg logging.
# All of these modules are just used by the ContextInjector below.
import inspect
import logging
import os
import socket
import traceback
psutil = None
try:
import psutil
except (OSError, ImportError):
# We run into issues when trying to import psutil from inside mod_wsgi on
# rhel7. If we hit that here, then just fail quietly.
# https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837
pass
class ContextInjector(logging.Filter):
""" Logging filter that adds context to log records.
Filters are typically used to "filter" log records. They declare a filter
method that can return True or False. Only records with 'True' will
actually be logged.
Here, we somewhat abuse the concept of a filter. We always return true,
but we use the opportunity to hang important contextual information on the
log record to later be used by the logging Formatter. We don't normally
want to see all this stuff in normal log records, but we *do* want to see
it when we are emailed error messages. Seeing an error, but not knowing
which host it comes from, is not that useful.
http://docs.python.org/2/howto/logging-cookbook.html#filters-contextual
"""
def filter(self, record):
current_process = ContextInjector.get_current_process()
current_hostname = socket.gethostname()
record.host = current_hostname
record.proc = current_process
record.pid = current_process.pid
record.proc_name = current_process.name
record.command_line = current_process.cmdline
# These are callabls on more modern versions of psutil.
if callable(record.proc_name):
record.proc_name = record.proc_name()
if callable(record.command_line):
record.command_line = record.command_line()
record.command_line = " ".join(record.command_line)
record.callstack = self.format_callstack()
return True
@staticmethod
def format_callstack():
for i, frame in enumerate(f[0] for f in inspect.stack()):
if not '__name__' in frame.f_globals:
continue
modname = frame.f_globals['__name__'].split('.')[0]
if modname != "logging":
break
def _format_frame(frame):
return ' File "%s", line %i in %s\n %s' % (frame)
stack = traceback.extract_stack()
stack = stack[:-i]
return "\n".join([_format_frame(frame) for frame in stack])
@staticmethod
def get_current_process():
mypid = os.getpid()
if not psutil:
raise OSError("Could not import psutil for %r" % mypid)
for proc in psutil.process_iter():
if proc.pid == mypid:
return proc
# This should be impossible.
raise ValueError("Could not find process %r" % mypid)
@classmethod
def __json__(cls):
""" We need to be jsonifiable for "fedmsg-config" """
return {'name': 'ContextInjector'}
hefty_format = """Message
-------
[%(asctime)s][%(name)10s %(levelname)7s]
%(message)s
Process Details
---------------
host: %(host)s
PID: %(pid)s
name: %(proc_name)s
command: %(command_line)s
Callstack that lead to the logging statement
--------------------------------------------
%(callstack)s
"""
# See the following for constraints on this format http://bit.ly/Xn1WDn
config = dict(
logging=dict(
version=1,
formatters=dict(
bare={
"datefmt": "%Y-%m-%d %H:%M:%S",
"format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s"
},
hefty={
"datefmt": "%Y-%m-%d %H:%M:%S",
"format": hefty_format,
},
),
filters=dict(
context={
# This "()" syntax in the stdlib doesn't seem to be documented
# anywhere. I had to read
# /usr/lib64/python2.7/logging/config.py to figure it out.
"()": ContextInjector,
},
),
handlers=dict(
console={
"class": "logging.StreamHandler",
"formatter": "bare",
"level": "INFO",
"stream": "ext://sys.stdout",
},
mailer={
"class": "logging.handlers.SMTPHandler",
"formatter": "hefty",
"filters": ["context"],
"level": "ERROR",
"mailhost": "bastion.vpn.fedoraproject.org",
"fromaddr": "fedmsg@fedoraproject.org",
"toaddrs": ["sysadmin-datanommer-members@fedoraproject.org"],
"subject": "fedmsg error log (anitya)",
},
),
loggers=dict(
fedmsg={
"level": "INFO",
"propagate": False,
"handlers": ["console", "mailer"],
},
moksha={
"level": "INFO",
"propagate": False,
"handlers": ["console", "mailer"],
},
),
),
)

View file

@ -1,18 +0,0 @@
config = dict(
active=True,
endpoints={
# This is the output side of the relay to which all other
# services can listen.
"relay_outbound": [
# Messages emerge here
#"tcp://release-monitoring.org:9940",
"tcp://anitya-frontend01.fedoraproject.org:9940",
],
},
# wsgi scripts on the frontend talk back here
# so do cronjobs on anitya-backend01. they get a firewall rule.
relay_inbound=[
"tcp://anitya-frontend01.fedoraproject.org:9941",
],
)

View file

@ -1,18 +0,0 @@
config = dict(
sign_messages=True,
validate_signatures=True,
ssldir="/etc/pki/fedmsg",
crl_location="https://fedoraproject.org/fedmsg/crl.pem",
crl_cache="/var/run/fedmsg/crl.pem",
crl_cache_expiry=86400, # Daily
certnames=dict([
("shell.anitya-frontend01", "shell-anitya-frontend01.vpn.fedoraproject.org"),
("anitya.anitya-frontend01", "anitya-anitya-frontend01.vpn.fedoraproject.org"),
("shell.anitya-backend01", "shell-anitya-backend01.vpn.fedoraproject.org"),
("anitya.anitya-backend01", "anitya-anitya-backend01.vpn.fedoraproject.org"),
]),
)

View file

@ -1,72 +0,0 @@
---
# Configuration for the anitya webapp
- name: install needed packages
package: name={{ item }} state=present update_cache=yes
with_items:
- anitya
- python-psycopg2
- python-memcached
- libsemanage-python
- httpd
- mod_ssl
tags:
- packages
- anitya_frontend
- name: Install all the configuration file of anitya
template: src={{ item.file }}
dest={{ item.location }}/{{ item.file }}
owner=apache group=apache mode=0600
with_items:
- { file: anitya.cfg, location: /etc/anitya }
- { file: anitya.conf, location: /etc/httpd/conf.d }
- { file: anitya.wsgi, location: /var/www/, dest: anitya.wsgi }
tags:
- config
- anitya_frontend
notify:
- restart apache
- name: create the folder where we store the ssl cert if not already there
file: state=directory
path=/etc/pki/tls/certs/
owner=root group=root mode=0755
- name: Install the SSL cert so that we can use https
copy: >
src={{ private}}/files/httpd/{{ item }} dest=/etc/pki/tls/certs/{{ item }}
owner=root group=root mode=0600
with_items:
- release-monitoring.org.cert
- release-monitoring.org.key
- release-monitoring.org.intermediate.cert
notify:
- restart apache
tags:
- config
- anitya_frontend
- name: Install the configuration file to activate https
template: >
src={{ item }} dest=/etc/httpd/conf.d/{{ item }}
owner=root group=root mode=0644
with_items:
- 0_releasemonitoring.conf
tags:
- files
- config
- anitya_frontend
notify:
- restart apache
- name: set sebooleans so anitya can talk to the db
seboolean: name={{ item }}
state=true
persistent=true
with_items:
- httpd_can_network_connect_db
- httpd_can_network_connect
tags:
- anitya_frontend

View file

@ -1,17 +0,0 @@
<VirtualHost *:80>
ServerName release-monitoring.org
Redirect permanent / https://release-monitoring.org/
</VirtualHost>
<VirtualHost *:443>
ServerName release-monitoring.org:443
SSLEngine on
SSLProtocol {{ ssl_protocols }}
SSLCipherSuite {{ ssl_ciphers }}
Header always add Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
SSLCertificateFile /etc/pki/tls/certs/release-monitoring.org.cert
SSLCertificateChainFile /etc/pki/tls/certs/release-monitoring.org.intermediate.cert
SSLCertificateKeyFile /etc/pki/tls/certs/release-monitoring.org.key
</VirtualHost>

View file

@ -1,44 +0,0 @@
# Beware that the quotes around the values are mandatory
from datetime import timedelta
# Set the time after which the session expires
PERMANENT_SESSION_LIFETIME = timedelta(hours=24)
### Secret key for the Flask application
SECRET_KEY='{{ anitya_secret_key }}'
### url to the database server:
#DB_URL=mysql://user:pass@host/db_name
#DB_URL=postgres://user:pass@host/db_name
DB_URL='postgresql://{{ anitya_db_user }}:{{ anitya_db_pass }}@{{ anitya_db_host }}/{{ anitya_db_name }}'
# List of admins based on their openid
ANITYA_WEB_ADMINS = [
'http://ralph.id.fedoraproject.org/',
'http://pingou.id.fedoraproject.org/',
'http://jcline.id.fedoraproject.org/',
'http://zlopez.id.fedoraproject.org/',
'http://tibbs.id.fedoraproject.org/',
'http://carlwgeorge.id.fedoraproject.org/',
]
# Email addresses to send tracebacks to when an HTTP 500 occurs
ADMIN_EMAIL = [
'admin@fedoraproject.org',
'jeremy@jcline.org',
]
# Fedora OpenID endpoint
{% if env == 'staging' %}
ANITYA_WEB_FEDORA_OPENID = 'https://id.stg.fedoraproject.org'
{% else %}
ANITYA_WEB_FEDORA_OPENID = 'https://id.fedoraproject.org'
{% endif %}
ANITYA_WEB_ALLOW_GOOGLE_OPENID = False
# This is required to fix login
PREFERRED_URL_SCHEME='https'
# Make browsers send session cookie only via HTTPS
SESSION_COOKIE_SECURE=True

View file

@ -1,23 +0,0 @@
Alias /static /usr/lib/python2.7/site-packages/anitya/static/
WSGIDaemonProcess anitya user=apache maximum-requests=1000 display-name=anitya processes=4 threads=4
WSGISocketPrefix run/wsgi
WSGIRestrictStdout Off
WSGIRestrictSignal Off
WSGIPythonOptimize 1
WSGIScriptAlias / /var/www/anitya.wsgi
<Location />
WSGIProcessGroup anitya
<IfModule mod_authz_core.c>
# Apache 2.4
Require all granted
</IfModule>
<IfModule !mod_authz_core.c>
# Apache 2.2
Order deny,allow
Allow from all
</IfModule>
</Location>

View file

@ -1,21 +0,0 @@
#-*- coding: UTF-8 -*-
# The three lines below are required to run on EL6 as EL6 has
# two possible version of python-sqlalchemy and python-jinja2
# These lines make sure the application uses the correct version.
import __main__
__main__.__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4']
import pkg_resources
import os
## Set the environment variable pointing to the configuration file
os.environ['ANITYA_WEB_CONFIG'] = '/etc/anitya/anitya.cfg'
## The following is only needed if you did not install anitya
## as a python module (for example if you run it from a git clone).
#import sys
#sys.path.insert(0, '/path/to/anitya/')
## The most import line to make the wsgi working
from anitya.app import APP as application

View file

@ -88,13 +88,6 @@ define service {
use defaulttemplate
}
define service {
hostgroup_name anitya-frontend
service_description Check for fedmsg-relay proc
check_command check_by_nrpe!check_fedmsg_relay_proc
use defaulttemplate
}
define service {
host_name badges-backend01.phx2.fedoraproject.org
service_description Check for fedmsg-hub proc
@ -252,12 +245,6 @@ define service {
check_command check_by_nrpe!check_datanommer_ansible
use defaulttemplate
}
define service {
host_name busgateway01.phx2.fedoraproject.org
service_description Check datanommer for recent anitya messages
check_command check_by_nrpe!check_datanommer_anitya
use defaulttemplate
}
define service {
host_name busgateway01.phx2.fedoraproject.org
service_description Check datanommer for recent fedimg messages
@ -358,13 +345,6 @@ define service {
use defaulttemplate
}
define service {
hostgroup_name anitya-frontend
service_description Check fedmsg consumers and producers relay
check_command check_by_nrpe!check_fedmsg_cp_anitya_relay
use defaulttemplate
}
define service {
host_name value01.phx2.fedoraproject.org
service_description Check fedmsg consumers and producers irc
@ -462,13 +442,6 @@ define service {
use defaulttemplate
}
define service {
hostgroup_name anitya-frontend
service_description Check fedmsg-relay consumers exceptions
check_command check_by_nrpe!check_fedmsg_cexceptions_anitya_relay
use defaulttemplate
}
define service {
host_name value01.phx2.fedoraproject.org
service_description Check fedmsg-irc consumers exceptions
@ -576,13 +549,6 @@ define service {
use defaulttemplate
}
define service {
hostgroup_name anitya-frontend
service_description Check fedmsg-relay consumers backlog
check_command check_by_nrpe!check_fedmsg_cbacklog_anitya_relay
use defaulttemplate
}
define service {
host_name value01.phx2.fedoraproject.org
service_description Check fedmsg-irc consumers backlog

View file

@ -10,7 +10,7 @@ scp db01.phx2.fedoraproject.org:/backups/pkgdb2-$(date +%F).dump.xz /srv/web/inf
scp db01.phx2.fedoraproject.org:/backups/koschei-$(date +%F).dump.xz /srv/web/infra/db-dumps/koschei.dump.xz
scp db01.phx2.fedoraproject.org:/backups/bodhi2-$(date +%F).dump.xz /srv/web/infra/db-dumps/bodhi2.dump.xz
scp db01.phx2.fedoraproject.org:/backups/pdc-$(date +%F).dump.xz /srv/web/infra/db-dumps/pdc.dump.xz
scp anitya-backend01.fedoraproject.org:/backups/anitya-$(date +%F).dump.xz /srv/web/infra/db-dumps/anitya.dump.xz
scp db01.phx2.fedoraproject.org:/backups/anitya-$(date +%F).dump.xz /srv/web/infra/db-dumps/anitya.dump.xz
scp db01.phx2.fedoraproject.org:/backups/mailman-$(date +%F).dump.xz /srv/web/infra/db-dumps/mailman.dump.xz
scp db01.phx2.fedoraproject.org:/backups/mbs-$(date +%F).dump.xz /srv/web/infra/db-dumps/mbs.dump.xz
scp db01.phx2.fedoraproject.org:/backups/odcs-$(date +%F).dump.xz /srv/web/infra/db-dumps/odcs.dump.xz