Retire the old datanommer & datagrepper
Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard
parent
4ad4c95a24
commit
726a788721
13 changed files with 2 additions and 471 deletions
|
|
@ -1,22 +0,0 @@
|
|||
---
|
||||
# Define resources for this group of hosts here.
|
||||
# Neeed for rsync from log01 for logs.
|
||||
custom_rules: ['-A INPUT -p tcp -m tcp -s 10.3.163.39 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT']
|
||||
deployment_type: prod
|
||||
freezes: false
|
||||
ipa_client_shell_groups:
|
||||
- fi-apprentice
|
||||
- sysadmin-datanommer
|
||||
- sysadmin-noc
|
||||
- sysadmin-veteran
|
||||
ipa_client_sudo_groups:
|
||||
- sysadmin-datanommer
|
||||
ipa_host_group: datagrepper
|
||||
ipa_host_group_desc: Service to grep through historical message bus data
|
||||
lvm_size: 20000
|
||||
mem_size: 8192
|
||||
num_cpus: 2
|
||||
primary_auth_source: ipa
|
||||
# for systems that do not match the above - specify the same parameter in
|
||||
# the host_vars/$hostname file
|
||||
tcp_ports: [80, 443, 6996]
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
---
|
||||
# Define resources for this group of hosts here.
|
||||
# Neeed for rsync from log01 for logs.
|
||||
custom_rules: ['-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT']
|
||||
freezes: false
|
||||
ipa_client_shell_groups:
|
||||
- fi-apprentice
|
||||
- sysadmin-datanommer
|
||||
- sysadmin-noc
|
||||
- sysadmin-veteran
|
||||
ipa_client_sudo_groups:
|
||||
- sysadmin-datanommer
|
||||
ipa_host_group: datagrepper
|
||||
ipa_host_group_desc: Service to grep through historical message bus data
|
||||
lvm_size: 20000
|
||||
mem_size: 2048
|
||||
num_cpus: 1
|
||||
# for systems that do not match the above - specify the same parameter in
|
||||
# the host_vars/$hostname file
|
||||
tcp_ports: [80, 443, 6996]
|
||||
|
|
@ -112,10 +112,10 @@ vmhost-a64-cc01.rdu-cc.fedoraproject.org
|
|||
|
||||
[datagrepper]
|
||||
#datagrepper01.iad2.fedoraproject.org
|
||||
datagrepper02.iad2.fedoraproject.org
|
||||
#datagrepper02.iad2.fedoraproject.org
|
||||
|
||||
[datagrepper_stg]
|
||||
datagrepper01.stg.iad2.fedoraproject.org
|
||||
#datagrepper01.stg.iad2.fedoraproject.org
|
||||
|
||||
[fedimg]
|
||||
fedimg01.iad2.fedoraproject.org
|
||||
|
|
@ -211,7 +211,6 @@ badges-web01.stg.iad2.fedoraproject.org
|
|||
blockerbugs01.stg.iad2.fedoraproject.org
|
||||
bodhi-backend01.stg.iad2.fedoraproject.org
|
||||
busgateway01.stg.iad2.fedoraproject.org
|
||||
datagrepper01.stg.iad2.fedoraproject.org
|
||||
koji01.stg.iad2.fedoraproject.org
|
||||
os-node01.stg.iad2.fedoraproject.org
|
||||
os-node02.stg.iad2.fedoraproject.org
|
||||
|
|
@ -637,7 +636,6 @@ copr-db-stg.aws.fedoraproject.org
|
|||
copr-dist-git-dev.aws.fedoraproject.org
|
||||
copr-fe-dev.aws.fedoraproject.org
|
||||
copr-keygen-dev.aws.fedoraproject.org
|
||||
datagrepper01.stg.iad2.fedoraproject.org
|
||||
db-datanommer01.stg.iad2.fedoraproject.org
|
||||
db-fas01.stg.iad2.fedoraproject.org
|
||||
db-koji01.stg.iad2.fedoraproject.org
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@
|
|||
roles:
|
||||
- role: fedmsg/hub
|
||||
enable_websocket_server: True
|
||||
- role: fedmsg/datanommer
|
||||
- role: fedmsg/relay
|
||||
- role: fedmsg/gateway
|
||||
- role: collectd/fedmsg-service
|
||||
|
|
|
|||
|
|
@ -1,108 +0,0 @@
|
|||
# create a new datagrepper server
|
||||
|
||||
- import_playbook: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=datagrepper:datagrepper_stg"
|
||||
|
||||
- name: make the box be real
|
||||
hosts: datagrepper:datagrepper_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- base
|
||||
- rkhunter
|
||||
- nagios_client
|
||||
- hosts
|
||||
- { role: openvpn/client,
|
||||
when: env != "staging" }
|
||||
- ipa/client
|
||||
- collectd/base
|
||||
- fedmsg/base
|
||||
- { role: rabbit/user,
|
||||
username: "datagrepper{{ env_suffix }}"}
|
||||
- rsyncd
|
||||
- sudo
|
||||
- mod_wsgi
|
||||
|
||||
pre_tasks:
|
||||
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
|
||||
|
||||
tasks:
|
||||
- import_tasks: "{{ tasks_path }}/motd.yml"
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
- name: dole out the service-specific config
|
||||
hosts: datagrepper:datagrepper_stg
|
||||
user: root
|
||||
gather_facts: True
|
||||
|
||||
vars_files:
|
||||
- /srv/web/infra/ansible/vars/global.yml
|
||||
- "/srv/private/ansible/vars.yml"
|
||||
- /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
|
||||
roles:
|
||||
- datagrepper
|
||||
- role: collectd/web-service
|
||||
site: datagrepper
|
||||
url: "http://localhost/datagrepper/raw?delta=86400"
|
||||
interval: 15
|
||||
|
||||
handlers:
|
||||
- import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
||||
# The gluster work here can be omitted for now. It is used by a feature of
|
||||
# datagrepper that is partially in place, but not yet functional.
|
||||
#
|
||||
#- name: set up gluster server on prod
|
||||
# hosts: datagrepper:datagrepper_stg
|
||||
# user: root
|
||||
# gather_facts: True
|
||||
#
|
||||
# vars_files:
|
||||
# - /srv/web/infra/ansible/vars/global.yml
|
||||
# - "/srv/private/ansible/vars.yml"
|
||||
# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
#
|
||||
# roles:
|
||||
# - role: gluster/server
|
||||
# name: gluster
|
||||
# username: "{{ datagrepperglusterusername }}"
|
||||
# password: "{{ datagrepperglusterpassword }}"
|
||||
# owner: root
|
||||
# group: root
|
||||
# datadir: /srv/glusterfs/datagrepper
|
||||
#
|
||||
# handlers:
|
||||
# - import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
#
|
||||
#- name: set up gluster client on prod
|
||||
# hosts: datagrepper:datagrepper_stg
|
||||
# user: root
|
||||
# gather_facts: True
|
||||
#
|
||||
# vars_files:
|
||||
# - /srv/web/infra/ansible/vars/global.yml
|
||||
# - "/srv/private/ansible/vars.yml"
|
||||
# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
|
||||
#
|
||||
# roles:
|
||||
# - role: gluster/client
|
||||
# name: gluster
|
||||
# servers:
|
||||
# - datagrepper01.iad2.fedoraproject.org
|
||||
# - datagrepper02.iad2.fedoraproject.org
|
||||
# username: "{{ datagrepperglusterusername }}"
|
||||
# password: "{{ datagrepperglusterpassword }}"
|
||||
# owner: apache
|
||||
# group: apache
|
||||
# mountdir: /var/cache/datagrepper
|
||||
#
|
||||
# handlers:
|
||||
# - import_tasks: "{{ handlers_path }}/restart_services.yml"
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
- name: install datagrepper and required packages
|
||||
package: name={{ item }} state=present
|
||||
with_items:
|
||||
- datagrepper
|
||||
- python-psycopg2
|
||||
- fedora-messaging
|
||||
tags:
|
||||
- packages
|
||||
- datagrepper
|
||||
|
||||
- name: creating datagrepper config dir
|
||||
file: path=/etc/datagrepper state=directory owner=root mode=755
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
- name: setting up datagrepper config from the template
|
||||
template: src=datagrepper.cfg dest=/etc/datagrepper/datagrepper.cfg owner=fedmsg group=fedmsg mode=640
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
- name: setting up credential from the template
|
||||
template: src=datagrepper-fedmsg.py dest=/etc/fedmsg.d/datagrepper.py owner=fedmsg group=fedmsg mode=640
|
||||
notify:
|
||||
- reload httpd
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
- name: remove bum compiled default config file
|
||||
file: dest="/etc/fedmsg.d/{{item}}" state=absent
|
||||
with_items:
|
||||
- datanommer.pyc
|
||||
- datanommer.pyo
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
# This file is in the package, empty, and we need to make sure we don't get that one.
|
||||
- name: empty out default config file
|
||||
copy:
|
||||
dest=/etc/fedmsg.d/datanommer.py
|
||||
content="config={}"
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
- name: installing memcached package for /topics
|
||||
package: name=python-memcached state=present
|
||||
tags:
|
||||
- packages
|
||||
- datagrepper
|
||||
|
||||
- name: setting up httpd config
|
||||
template: src=datagrepper-app.conf dest=/etc/httpd/conf.d/datagrepper.conf owner=root group=root
|
||||
notify:
|
||||
- reload httpd
|
||||
tags:
|
||||
- config
|
||||
- datagrepper
|
||||
|
||||
- name: ensure selinux lets httpd talk to the database
|
||||
seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
|
||||
tags:
|
||||
- datagrepper
|
||||
|
||||
- name: ensure selinux lets httpd talk to the memcache
|
||||
seboolean: name=httpd_can_network_memcache persistent=yes state=yes
|
||||
tags:
|
||||
- datagrepper
|
||||
|
||||
# selinux policy has been intentionally omitted since that is obtained from fedmsg/base
|
||||
|
||||
- name: Create /etc/pki/fedora-messaging
|
||||
file:
|
||||
dest: /etc/pki/fedora-messaging
|
||||
mode: 0775
|
||||
owner: root
|
||||
group: root
|
||||
state: directory
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: Deploy the Fedora datagrepper fedora-messaging cert
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt"
|
||||
dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem
|
||||
mode: 0644
|
||||
owner: root
|
||||
group: root
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
||||
- name: Deploy the Fedora datagrepper fedora-messaging key
|
||||
copy:
|
||||
src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key"
|
||||
dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem
|
||||
mode: 0640
|
||||
owner: root
|
||||
group: root
|
||||
when: "deployment_type is defined"
|
||||
tags:
|
||||
- config
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
LoadModule expires_module modules/mod_expires.so
|
||||
LoadModule headers_module modules/mod_headers.so
|
||||
LoadModule deflate_module modules/mod_deflate.so
|
||||
|
||||
ExpiresActive On
|
||||
#ExpiresDefault "access plus 300 seconds"
|
||||
|
||||
ErrorLog logs/datagrepper_error_log
|
||||
CustomLog logs/datagrepper_access_log combined
|
||||
|
||||
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/javascript text/css application/x-javascript
|
||||
|
||||
# This caching may not necessarily be ideal, or even correct.
|
||||
# However, it was the only I could get firebug to show me 302's for
|
||||
# my ToscaWidget resources.
|
||||
<filesmatch ".(gif|jpe?g|png|css|js)$">
|
||||
Header unset Cache-Control
|
||||
Header unset Etag
|
||||
Header add Cache-Control "max-age=2592000"
|
||||
#ExpiresDefault A2592000
|
||||
</filesmatch>
|
||||
|
||||
# Static resources for the datagrepper app.
|
||||
Alias /datagrepper/static/ /usr/lib/python2.7/site-packages/datagrepper/static/
|
||||
|
||||
WSGIDaemonProcess datagrepper user=fedmsg group=fedmsg maximum-requests=50000 display-name=datagrepper processes=20 threads=5 inactivity-timeout=900
|
||||
WSGISocketPrefix run/wsgi
|
||||
WSGIRestrictStdout Off
|
||||
WSGIRestrictSignal Off
|
||||
WSGIPythonOptimize 1
|
||||
|
||||
WSGIScriptAlias /datagrepper /usr/share/datagrepper/apache/datagrepper.wsgi
|
||||
|
||||
<Directory /usr/lib/python2.7/site-packages/datagrepper/static/>
|
||||
Order deny,allow
|
||||
Require all granted
|
||||
</Directory>
|
||||
<Directory /usr/share/datagrepper/apache/>
|
||||
WSGIProcessGroup datagrepper
|
||||
Order deny,allow
|
||||
Require all granted
|
||||
</Directory>
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# Configuration for the datagrepper webapp.
|
||||
config = {
|
||||
# We don't actually want to run the datanommer consumer on this machine.
|
||||
'datanommer.enabled': False,
|
||||
|
||||
# Note that this is connecting to db02. That's fine for now, but we want to
|
||||
# move the db for datanommer to a whole other db host in the future. We
|
||||
# expect the amount of data it generates to grow pretty steadily over time
|
||||
# and we don't want *read* operations on that database to slow down all our
|
||||
# other apps.
|
||||
{% if env == "staging" %}
|
||||
'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer',
|
||||
'fedmenu_url': 'https://apps.stg.fedoraproject.org/fedmenu',
|
||||
'fedmenu_data_url': 'https://apps.stg.fedoraproject.org/js/data.js',
|
||||
{% else %}
|
||||
'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.iad2.fedoraproject.org/datanommer',
|
||||
'fedmenu_url': 'https://apps.fedoraproject.org/fedmenu',
|
||||
'fedmenu_data_url': 'https://apps.fedoraproject.org/js/data.js',
|
||||
{% endif %}
|
||||
|
||||
# Only allow ajax/websockets connections back to our domains.
|
||||
# https://github.com/fedora-infra/datagrepper/pull/192
|
||||
'content_security_policy': 'connect-src https://*.fedoraproject.org wss://*.fedoraproject.org'
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
from datetime import timedelta
|
||||
|
||||
### Secret key for the Flask application
|
||||
SECRET_KEY = '{{ datagrepperCookieSecret }}'
|
||||
|
||||
### Unhappy mako
|
||||
MAKO_OUTPUT_ENCODING='utf-8'
|
||||
|
||||
{% if env == "staging" %}
|
||||
DATAGREPPER_BASE_URL='https://apps.stg.fedoraproject.org/datagrepper/'
|
||||
{% else %}
|
||||
DATAGREPPER_BASE_URL='https://apps.fedoraproject.org/datagrepper/'
|
||||
{% endif %}
|
||||
|
||||
DATAGREPPER_CACHE_BACKEND='dogpile.cache.memcached'
|
||||
DATAGREPPER_CACHE_KWARGS={'arguments': {'url': 'memcached01:11211'}}
|
||||
|
||||
{% if env == "staging" %}
|
||||
SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.stg.iad2.fedoraproject.org:5432/datagrepper'
|
||||
{% else %}
|
||||
SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.iad2.fedoraproject.org:5432/datagrepper'
|
||||
{% endif %}
|
||||
|
||||
{% if env == "staging" %}
|
||||
DATAGREPPER_OPENID_ENDPOINT='id.stg.fedoraproject.org'
|
||||
{% else %}
|
||||
DATAGREPPER_OPENID_ENDPOINT='id.fedoraproject.org'
|
||||
{% endif %}
|
||||
|
||||
RUNNER_LOCKFILE='/var/run/fedmsg/datagrepper.lock'
|
||||
JOB_OUTPUT_DIR='/var/cache/datagrepper'
|
||||
JOB_OUTPUT_URL='https://apps.{% if env == "staging" %}stg.{% endif %}fedoraproject.org/datagrepper/output'
|
||||
JOB_EXPIRY=timedelta(days=7)
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# This is a *very* simple role. The config needed for fedmsg-relay to operate
|
||||
# correctly is actually included as part of the fedmsg/base role.
|
||||
|
||||
- name: install datanommer packages
|
||||
package:
|
||||
state: present
|
||||
name:
|
||||
- python-datanommer-consumer
|
||||
- python-datanommer-models
|
||||
- python-psycopg2
|
||||
- datanommer-commands
|
||||
tags:
|
||||
- packages
|
||||
- datanommer
|
||||
|
||||
- name: copy alembic config
|
||||
template: >
|
||||
src=alembic.ini
|
||||
dest=/usr/share/datanommer.models/alembic.ini
|
||||
mode=0640
|
||||
tags:
|
||||
- config
|
||||
- datanommer
|
||||
notify:
|
||||
- restart fedmsg-hub
|
||||
|
||||
- name: copy datanommer config itself
|
||||
template: >
|
||||
src="{{ item }}"
|
||||
dest="/etc/fedmsg.d/{{item}}"
|
||||
mode=0640
|
||||
owner=fedmsg
|
||||
group=nrpe
|
||||
with_items:
|
||||
- datanommer.py
|
||||
- fas-credentials.py
|
||||
tags:
|
||||
- config
|
||||
- datanommer
|
||||
notify:
|
||||
- restart fedmsg-hub
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = /usr/share/datanommer.models/alembic/
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
{% if env == "production" %}
|
||||
sqlalchemy.url = postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01/datanommer
|
||||
{% else %}
|
||||
sqlalchemy.url = postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer
|
||||
{% endif %}
|
||||
fedmsg_config_dir = /etc/fedmsg.d/
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# Configuration for the datanommer consumer. A plugin for the fedmsg-hub process.
|
||||
config = {
|
||||
'datanommer.enabled': True,
|
||||
{% if env == "production" %}
|
||||
'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.iad2.fedoraproject.org/datanommer',
|
||||
{% else %}
|
||||
'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommer_stg_db_password }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer',
|
||||
{% endif %}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
config = dict(
|
||||
fas_credentials=dict(
|
||||
username="fedoradummy",
|
||||
password="{{ fedoraDummyUserPassword }}",
|
||||
{% if env == 'staging' %}
|
||||
base_url="https://accounts.stg.fedoraproject.org/",
|
||||
{% endif %}
|
||||
),
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue