[odcs] Add odcs playbook and roles

playbooks/groups/odcs.yml

@@ -46,38 +46,6 @@
   handlers:
   - include: "{{ handlers_path }}/restart_services.yml"
 
-- name: Set up apache on the frontend MBS API app
-  hosts: odcs-frontend:odcs-frontend-stg
-  user: root
-  gather_facts: True
-
-  vars_files:
-   - /srv/web/infra/ansible/vars/global.yml
-   - "/srv/private/ansible/vars.yml"
-   - "{{ vars_path }}/{{ ansible_distribution }}.yml"
-
-  roles:
-  - mod_wsgi
-
-  handlers:
-  - include: "{{ handlers_path }}/restart_services.yml"
-
-- name: set up fedmsg configuration and common odcs files
-  hosts: odcs:odcs-stg
-  user: root
-  gather_facts: True
-
-  vars_files:
-   - /srv/web/infra/ansible/vars/global.yml
-   - "/srv/private/ansible/vars.yml"
-   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
-
-  roles:
-  - fedmsg/base
-
-  handlers:
-  - include: "{{ handlers_path }}/restart_services.yml"
-
 - name: Set up a gluster share on the backend for the frontend
   hosts: odcs:odcs-stg
   user: root
@@ -122,3 +90,36 @@
     group: root
     mountdir: /srv/odcs
     when: env != 'staging'
+
+- name: Set up odcs frontend service
+  hosts: odcs-frontend:odcs-frontend-stg
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - "{{ vars_path }}/{{ ansible_distribution }}.yml"
+
+  roles:
+  - mod_wsgi
+  - odcs/frontend
+
+  handlers:
+  - include: "{{ handlers_path }}/restart_services.yml"
+
+- name: set up odcs backend service
+  hosts: odcs-backend:odcs-backend-stg
+  user: root
+  gather_facts: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  roles:
+  - odcs/backend
+
+  handlers:
+  - include: "{{ handlers_path }}/restart_services.yml"

roles/odcs/backend/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+    - { role: odcs/base }

roles/odcs/backend/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: generate the ODCS koji config
+  template:
+    src: etc/koji.conf.d/odcs.conf.j2
+    dest: /etc/koji.conf.d/odcs.conf
+    owner: odcs
+    group: odcs
+    mode: 0440
+  notify:
+  - restart odcs-backend
+  tags:
+  - odcs

roles/odcs/backend/templates/etc/koji.conf.d/odcs.conf.j2

@@ -0,0 +1,13 @@
+[production]
+server = https://koji.fedoraproject.org/kojihub
+weburl = https://koji.fedoraproject.org/koji
+topurl = https://kojipkgs.fedoraproject.org/
+authtype = kerberos
+krb_rdns = false
+
+[staging]
+server = https://koji.stg.fedoraproject.org/kojihub
+weburl = https://koji.stg.fedoraproject.org/koji
+topurl = https://kojipkgs.stg.fedoraproject.org/
+authtype = kerberos
+krb_rdns = false

roles/odcs/base/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+odcs_upgrade: False
+odcs_force_postgres_ssl: False
+odcs_pdc_insecure: False
+odcs_pdc_develop: True
+odcs_target_dir: /srv/odcs
+odcs_target_dir_url: http://{{ inventory_hostname }}/composes
+odcs_allowed_clients_groups: ["sysadmin-mbs"]
+odcs_allowed_clients_users: []
+odcs_admin_groups: ["sysadmin-mbs"]
+odcs_admin_users: []

roles/odcs/base/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: restart odcs-backend
+  service:
+    name: "odcs-backend"
+    state: restarted

roles/odcs/base/tasks/main.yml

@@ -0,0 +1,77 @@
+---
+# install packages and generate shared configuration files
+- name: install the packages required for ODCS
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - python-psycopg2
+  - python2-odcs-common
+  - odcs
+  tags:
+  - odcs
+
+- name: install the latest ODCS packagess
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+  - python2-odcs-common
+  - odcs
+  when: odcs_upgrade
+  tags:
+  - odcs
+
+# install required packages for frontend here, as we may
+# need to reload httpd in next task when host is frontend
+- name: install the packages required for ODCS frontend
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - mod_auth_openidc
+  - mod_wsgi
+  when: inventory_hostname.startswith('odcs-frontend')
+  tags:
+  - odcs
+
+- name: install the packages required for ODCS backend
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+  - koji
+  when: inventory_hostname.startswith('odcs-backend')
+  tags:
+  - odcs
+
+# this app config is shared by backend and frontend, but has different
+# owner groups on backend and frontend, and notify different handlers,
+# we can have vars set for frontend and backend seperately to do that,
+# but it looks a little weird to have such special vars in
+# inventory/group_vars/odcs-*, so just write 2 tasks to keep it simple.
+- name: generate the ODCS app config for frontend
+  template:
+    src: etc/odcs/config.py.j2
+    dest: /etc/odcs/config.py
+    owner: odcs
+    group: apache
+    mode: 0440
+  notify:
+  - restart apache
+  when: inventory_hostname.startswith('odcs-frontend')
+  tags:
+  - odcs
+
+- name: generate the ODCS app config for backend
+  template:
+    src: etc/odcs/config.py.j2
+    dest: /etc/odcs/config.py
+    owner: odcs
+    group: odcs
+    mode: 0440
+  notify:
+  - restart odcs-backend
+  when: inventory_hostname.startswith('odcs-backend')
+  tags:
+  - odcs

roles/odcs/base/templates/etc/odcs/config.py.j2

@@ -0,0 +1,101 @@
+from os import path
+
+confdir = path.abspath(path.dirname(__file__))
+# use parent dir as dbdir else fallback to current dir
+dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \
+    else confdir
+
+class BaseConfiguration(object):
+    # Make this random (used to generate session keys)
+    SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join(
+        dbdir, 'odcs.db'))
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+
+    HOST = '127.0.0.1'
+    PORT = 5005
+
+    DEBUG = False
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 120
+    NET_RETRY_INTERVAL = 30
+
+    # Available backends are: console, file, journal.
+    LOG_BACKEND = 'journal'
+
+    # Path to log file when LOG_BACKEND is set to "file".
+    LOG_FILE = 'odcs.log'
+
+    # Available log levels are: debug, info, warn, error.
+    LOG_LEVEL = 'info'
+
+    SSL_ENABLED = False
+
+
+class DevConfiguration(BaseConfiguration):
+    DEBUG = True
+    LOG_BACKEND = 'console'
+    LOG_LEVEL = 'debug'
+
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 5
+    NET_RETRY_INTERVAL = 1
+
+
+class TestConfiguration(BaseConfiguration):
+    LOG_BACKEND = 'console'
+    LOG_LEVEL = 'debug'
+    DEBUG = True
+
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(
+        path.join(dbdir, 'tests', 'test_odcs.db'))
+
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 3
+    NET_RETRY_INTERVAL = 1
+
+
+class ProdConfiguration(BaseConfiguration):
+    AUTH_BACKEND = 'openidc'
+    AUTH_OPENIDC_REQUIRED_SCOPES = [
+        'openid',
+        'https://id.fedoraproject.org/scope/groups',
+        'https://pagure.io/odcs/new-compose',
+        'https://pagure.io/odcs/renew-compose',
+        'https://pagure.io/odcs/delete-compose',
+    ]
+
+{% if env == 'staging' %}
+    AUTH_OPENIDC_USERINFO_URI = 'https://iddev.fedorainfracloud.org/openidc/UserInfo'
+    SECRET_KEY = "{{ odcs_stg_secret_key }}"
+
+    SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://odcs:{{odcs_stg_db_password}}@db-odcs/odcs{{ '?sslmode=require' if odcs_force_postgres_ssl else '' }}'
+
+    KOJI_PROFILE = 'staging'
+
+    PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1'
+{% else %}
+    AUTH_OPENIDC_USERINFO_URI = 'https://id.fedoraproject.org/openidc/UserInfo'
+    SECRET_KEY = "{{ odcs_prod_secret_key }}"
+
+    SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://odcs:{{odcs_prod_db_password}}@db-odcs/odcs{{ '?sslmode=require' if odcs_force_postgres_ssl else '' }}'
+
+    KOJI_PROFILE = 'production'
+
+    PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'
+{% endif %}
+    PDC_INSECURE = {{ odcs_pdc_insecure }}
+    PDC_DEVELOP = {{ odcs_pdc_develop }}
+
+    TARGET_DIR = "{{ odcs_target_dir }}"
+    TARGET_DIR_URL = "{{ odcs_target_dir_url }}"
+
+    ALLOWED_CLIENTS = {
+        'groups': {{ odcs_allowed_clients_groups }},
+        'users': {{ odcs_allowed_clients_users }},
+    }
+
+    ADMINS = {
+        'groups': {{ odcs_admin_groups }},
+        'users': {{ odcs_admin_users }},
+    }

roles/odcs/frontend/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+odcs_migrate_db: True
+odcs_endpoint: ''
+odcs_allowed_named_hosts: []
+odcs_allowed_hosts: []
+odcs_force_ssl: True
+odcs_target_dir: /var/www/composes

roles/odcs/frontend/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+    - { role: odcs/base }

roles/odcs/frontend/tasks/main.yml

@@ -0,0 +1,44 @@
+---
+- name: modify selinux so that httpd can serve data from NFS shares if needed
+  seboolean:
+    name: httpd_use_nfs
+    state: yes
+    persistent: yes
+  when: "'enabled' in ansible_selinux.status"
+  tags:
+  - odcs
+  - selinux
+
+- name: create ODCS_TARGET_DIR
+  file:
+    path: "{{ odcs_target_dir }}"
+    state: directory
+    owner: apache
+    group: apache
+    mode: 0775
+    recurse: yes
+
+- name: generate the ODCS Apache config
+  template:
+    src: etc/httpd/conf.d/odcs.conf.j2
+    dest: /etc/httpd/conf.d/odcs.conf
+    owner: apache
+    group: apache
+    mode: 0440
+  notify:
+  - reload apache
+  tags:
+  - odcs
+
+# This will initialize Alembic if the database is empty, and migrate to the
+# latest revision
+- name: migrate the database
+  command: "{{ item }}"
+  with_items:
+  - odcs-manager upgradedb
+  - odcs-manager db migrate
+  become: yes
+  become_user: odcs
+  when: odcs_migrate_db
+  tags:
+  - odcs

roles/odcs/frontend/templates/etc/httpd/conf.d/odcs.conf.j2

@@ -0,0 +1,60 @@
+{% if odcs_force_ssl %}
+# Force SSL
+RewriteEngine On
+RewriteCond %{HTTPS} off
+RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
+{% endif %}
+
+WSGIDaemonProcess odcs user=odcs group=odcs threads=5
+WSGIScriptAlias /{{ odcs_endpoint }} /usr/share/odcs/odcs.wsgi
+
+{% if env == 'staging' %}
+OIDCOAuthClientID {{ odcs_stg_oidc_client_id }}
+OIDCOAuthClientSecret {{ odcs_stg_oidc_client_secret }}
+{% else %}
+OIDCOAuthClientID {{ odcs_prod_oidc_client_id }}
+OIDCOAuthClientSecret {{ odcs_prod_oidc_client_secret }}
+{% endif %}
+
+OIDCOAuthIntrospectionEndpoint https://id.fedoraproject.org/openidc/TokenInfo
+OIDCOAuthIntrospectionEndpointAuth client_secret_post
+OIDCOAuthIntrospectionEndpointParams token_type_hint=Bearer
+
+<Directory /usr/share/odcs>
+    WSGIProcessGroup odcs
+    WSGIApplicationGroup %{GLOBAL}
+
+    <RequireAll>
+        {% if odcs_allowed_named_hosts or odcs_allowed_hosts %}
+        <RequireAny>
+            Require method GET POST DELETE
+            {% if odcs_allowed_named_hosts %}
+            {{ 'Require host ' ~ odcs_allowed_named_hosts|join(' ') }}
+            {% endif %}
+            {% if odcs_allowed_hosts %}
+            {{ 'Require ip ' ~ odcs_allowed_hosts|join(' ') }}
+            {% endif %}
+        </RequireAny>
+        {% endif %}
+        Require all granted
+    </RequireAll>
+</Directory>
+
+Alias "/composes" "{{ odcs_target_dir }}"
+<Directory {{ odcs_target_dir }}>
+    {% if odcs_allowed_named_hosts or odcs_allowed_hosts %}
+    <RequireAny>
+        Require method GET
+        {% if odcs_allowed_named_hosts %}
+        {{ 'Require host ' ~ odcs_allowed_named_hosts|join(' ') }}
+        {% endif %}
+        {% if odcs_allowed_hosts -%}
+        {{ 'Require ip ' ~ odcs_allowed_hosts|join(' ') }}
+        {% endif %}
+    </RequireAny>
+    {% else %}
+    Require all granted
+    {% endif %}
+
+    Options +Indexes
+</Directory>