Import the datagrepper role from the main ansible repo

Signed-off-by: Pierre-Yves Chibon <pingou@pingoured.fr>

ansible/roles/datagrepper/tasks/main.yml

@@ -0,0 +1,106 @@
+- name: install datagrepper and required packages
+  package: name={{ item }} state=present
+  with_items:
+  - datagrepper
+  - python-psycopg2
+  - fedora-messaging
+  tags:
+  - packages
+  - datagrepper
+
+- name: creating datagrepper config dir
+  file: path=/etc/datagrepper state=directory owner=root mode=755
+  tags:
+  - config
+  - datagrepper
+
+- name: setting up datagrepper config from the template
+  template: src=datagrepper.cfg dest=/etc/datagrepper/datagrepper.cfg owner=fedmsg group=fedmsg mode=640
+  tags:
+  - config
+  - datagrepper
+
+- name: setting up credential from the template
+  template: src=datagrepper-fedmsg.py dest=/etc/fedmsg.d/datagrepper.py owner=fedmsg group=fedmsg mode=640
+  notify:
+  - reload httpd
+  tags:
+  - config
+  - datagrepper
+
+- name: remove bum compiled default config file
+  file: dest="/etc/fedmsg.d/{{item}}" state=absent
+  with_items:
+  - datanommer.pyc
+  - datanommer.pyo
+  tags:
+  - config
+  - datagrepper
+
+# This file is in the package, empty, and we need to make sure we don't get that one.
+- name: empty out default config file
+  copy:
+    dest=/etc/fedmsg.d/datanommer.py
+    content="config={}"
+  tags:
+  - config
+  - datagrepper
+
+- name: installing memcached package for /topics
+  package: name=python-memcached state=present
+  tags:
+  - packages
+  - datagrepper
+
+- name: setting up httpd config
+  template: src=datagrepper-app.conf dest=/etc/httpd/conf.d/datagrepper.conf owner=root group=root
+  notify:
+  - reload httpd
+  tags:
+  - config
+  - datagrepper
+
+- name: ensure selinux lets httpd talk to the database
+  seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
+  tags:
+  - datagrepper
+
+- name: ensure selinux lets httpd talk to the memcache
+  seboolean: name=httpd_can_network_memcache persistent=yes state=yes
+  tags:
+  - datagrepper
+
+# selinux policy has been intentionally omitted since that is obtained from fedmsg/base
+
+- name: Create /etc/pki/fedora-messaging
+  file:
+    dest: /etc/pki/fedora-messaging
+    mode: 0775
+    owner: root
+    group: root
+    state: directory
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging cert
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt"
+    dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem
+    mode: 0644
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging key
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key"
+    dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem
+    mode: 0640
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config

ansible/roles/datagrepper/templates/datagrepper-fedmsg.py

@@ -0,0 +1,24 @@
+# Configuration for the datagrepper webapp.
+config = {
+    # We don't actually want to run the datanommer consumer on this machine.
+    'datanommer.enabled': False,
+
+    # Note that this is connecting to db02.  That's fine for now, but we want to
+    # move the db for datanommer to a whole other db host in the future.  We
+    # expect the amount of data it generates to grow pretty steadily over time
+    # and we don't want *read* operations on that database to slow down all our
+    # other apps.
+    {% if env == "staging" %}
+    'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.stg.iad2.fedoraproject.org/datanommer',
+    'fedmenu_url': 'https://apps.stg.fedoraproject.org/fedmenu',
+    'fedmenu_data_url': 'https://apps.stg.fedoraproject.org/js/data.js',
+    {% else %}
+    'datanommer.sqlalchemy.url': 'postgresql://{{ datanommerDBUser }}:{{ datanommerDBPassword }}@db-datanommer01.iad2.fedoraproject.org/datanommer',
+    'fedmenu_url': 'https://apps.fedoraproject.org/fedmenu',
+    'fedmenu_data_url': 'https://apps.fedoraproject.org/js/data.js',
+    {% endif %}
+
+    # Only allow ajax/websockets connections back to our domains.
+    # https://github.com/fedora-infra/datagrepper/pull/192
+    'content_security_policy': 'connect-src https://*.fedoraproject.org wss://*.fedoraproject.org'
+}

ansible/roles/datagrepper/templates/datagrepper.cfg

@@ -0,0 +1,33 @@
+from datetime import timedelta
+
+### Secret key for the Flask application
+SECRET_KEY = '{{ datagrepperCookieSecret }}'
+
+### Unhappy mako
+MAKO_OUTPUT_ENCODING='utf-8'
+
+{% if env == "staging" %}
+DATAGREPPER_BASE_URL='https://apps.stg.fedoraproject.org/datagrepper/'
+{% else %}
+DATAGREPPER_BASE_URL='https://apps.fedoraproject.org/datagrepper/'
+{% endif %}
+
+DATAGREPPER_CACHE_BACKEND='dogpile.cache.memcached'
+DATAGREPPER_CACHE_KWARGS={'arguments': {'url': 'memcached01:11211'}}
+
+{% if env == "staging" %}
+SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.stg.iad2.fedoraproject.org:5432/datagrepper'
+{% else %}
+SQLALCHEMY_DATABASE_URI='postgresql+psycopg2://{{ datagrepper_app_user }}:{{ datagrepper_app_password }}@db-datanommer01.iad2.fedoraproject.org:5432/datagrepper'
+{% endif %}
+
+{% if env == "staging" %}
+DATAGREPPER_OPENID_ENDPOINT='id.stg.fedoraproject.org'
+{% else %}
+DATAGREPPER_OPENID_ENDPOINT='id.fedoraproject.org'
+{% endif %}
+
+RUNNER_LOCKFILE='/var/run/fedmsg/datagrepper.lock'
+JOB_OUTPUT_DIR='/var/cache/datagrepper'
+JOB_OUTPUT_URL='https://apps.{% if env == "staging" %}stg.{% endif %}fedoraproject.org/datagrepper/output'
+JOB_EXPIRY=timedelta(days=7)