First draft of the mbs/frontend role.

playbooks/groups/mbs.yml

@@ -62,7 +62,7 @@
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
 
-- name: set up fedmsg configuration
+- name: set up fedmsg configuration and common mbs files
   hosts: mbs:mbs-stg
   user: root
   gather_facts: True
@@ -74,6 +74,7 @@
 
   roles:
   - fedmsg/base
+  - mbs/common
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
@@ -89,7 +90,7 @@
    - "{{ vars_path }}/{{ ansible_distribution }}.yml"
 
   roles:
-  #- mbs/frontend
+  - mbs/frontend
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
@@ -105,6 +106,10 @@
    - "{{ vars_path }}/{{ ansible_distribution }}.yml"
 
   roles:
+  - role: keytab/service
+    service: mbs
+    owner_user: fedmsg
+    host: "mbs{{env_suffix}}.fedoraproject.org"
   - fedmsg/hub
   #- mbs/backend
 

roles/mbs/common/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+# Common configuration for the Module Build Service (MBS) pieces
+
+- name: install needed packages
+  yum: pkg={{ item }} state=present
+  with_items:
+  - module-build-service
+  - python-psycopg2
+  - libsemanage-python
+  - python-memcached
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common
+
+- name: copy app configuration
+  template: >
+    src=config.py dest=/etc/module-build-service/config.py
+    owner=root group=fedmsg mode=0640
+  notify:
+  - restart apache
+  - restart fedmsg-hub
+  tags:
+  - mbs
+  - mbs/common

roles/mbs/common/templates/config.py

@@ -0,0 +1,119 @@
+from os import path
+
+# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are
+# declared properly somewhere/somehow
+confdir = path.abspath(path.dirname(__file__))
+# use parent dir as dbdir else fallback to current dir
+dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \
+        else confdir
+
+
+class BaseConfiguration(object):
+    DEBUG = False
+    # Make this random (used to generate session keys)
+    SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join(
+        dbdir, 'module_build_service.db'))
+    SQLALCHEMY_TRACK_MODIFICATIONS = True
+    # Where we should run when running "manage.py runssl" directly.
+    HOST = '0.0.0.0'
+    PORT = 5000
+
+    # Global network-related values, in seconds
+    NET_TIMEOUT = 120
+    NET_RETRY_INTERVAL = 30
+
+    SYSTEM = 'koji'
+    MESSAGING = 'fedmsg'  # or amq
+    MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod']
+    KOJI_CONFIG = '/etc/module-build-service/koji.conf'
+    KOJI_PROFILE = 'koji'
+    KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64']
+    KOJI_PROXYUSER = True
+    KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos'
+    COPR_CONFIG = '/etc/module-build-service/copr.conf'
+    PDC_URL = 'http://modularity.fedorainfracloud.org:8080/rest_api/v1'
+    PDC_INSECURE = True
+    PDC_DEVELOP = True
+    SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"]
+
+    # How often should we resort to polling, in seconds
+    # Set to zero to disable polling
+    POLLING_INTERVAL = 600
+
+    # Determines how many builds that can be submitted to the builder
+    # and be in the build state at a time. Set this to 0 for no restrictions
+    NUM_CONSECUTIVE_BUILDS = 5
+
+    RPMS_DEFAULT_REPOSITORY = 'git://pkgs.fedoraproject.org/rpms/'
+    RPMS_ALLOW_REPOSITORY = False
+    RPMS_DEFAULT_CACHE = 'http://pkgs.fedoraproject.org/repo/pkgs/'
+    RPMS_ALLOW_CACHE = False
+
+    # Available backends are: console, file, journal.
+    LOG_BACKEND = 'journal'
+
+    # Path to log file when LOG_BACKEND is set to "file".
+    LOG_FILE = 'module_build_service.log'
+
+    # Available log levels are: debug, info, warn, error.
+    LOG_LEVEL = 'info'
+
+    # Settings for Kerberos
+    KRB_KEYTAB = None
+    KRB_PRINCIPAL = None
+    KRB_CCACHE = None
+
+    # AMQ prefixed variables are required only while using 'amq' as messaging backend
+    # Addresses to listen to
+    AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji',
+                          'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service']
+    # Address for sending messages
+    AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'
+    AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt'
+    AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key'
+    AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt'
+
+
+class ProdConfiguration(BaseConfiguration):
+    DEBUG = False  # Don't turn this on.
+
+    {% if env == 'staging' %}
+    SECRET_KEY = '{{ mbs_stg_secret_key }}'
+    SQLALCHEMY_DATABASE_URI = 'postgresql://mbs:{{mbs_stg_db_password}}@db-mbs/mbs'
+    {% else %}
+    SECRET_KEY = '{{ mbs_prod_secret_key }}'
+    SQLALCHEMY_DATABASE_URI = 'postgresql://mbs:{{mbs_prod_db_password}}@db-mbs/mbs'
+    {% endif %}
+
+    {% if env == 'staging' %}
+    KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG'
+    {% else %}
+    KRB_PRINCIPAL = 'modularity@FEDORAPROJECT.ORG'
+    {% endif %}
+
+    KRB_KEYTAB = '/etc/krb5.mbs_mbs{{env_suffix}}.fedoraproject.org.keytab'
+    KRB_CCACHE = '/var/cache/mbs-krb5cc'
+
+    LOG_LEVEL = 'debug'
+    LOG_BACKEND = 'console'
+
+    PDC_INSECURE = False
+    PDC_DEVELOP = False
+
+    KOJI_CONFIG = path.join(confdir, 'koji.conf')
+    {% if env == 'staging' %}
+    KOJI_PROFILE = 'staging'
+    KOJI_ARCHES = ['x86_64']
+    KOJI_REPOSITORY_URL = 'http://kojipkgs.stg.fedoraproject.org/repos'
+    MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.stg']
+    PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1'
+    SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"]
+    {% else %}
+    KOJI_PROFILE = 'production'
+    KOJI_ARCHES = ['x86_64']
+    KOJI_REPOSITORY_URL = 'http://kojipkgs.fedoraproject.org/repos'
+    MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod']
+    PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'
+    SCMURLS = ["git://pkgs.fedoraproject.org/modules/"]
+    {% endif %}

roles/mbs/common/templates/koji.conf

@@ -0,0 +1,13 @@
+[production]
+server = https://koji.fedoraproject.org/kojihub
+weburl = https://koji.fedoraproject.org/koji
+topurl = https://kojipkgs.fedoraproject.org/
+authtype = kerberos
+krb_rdns = false
+
+[staging]
+server = https://koji.stg.fedoraproject.org/kojihub
+weburl = https://koji.stg.fedoraproject.org/koji
+topurl = https://kojipkgs.stg.fedoraproject.org/
+authtype = kerberos
+krb_rdns = false

roles/mbs/frontend/files/mbs-scheduler.py

@@ -0,0 +1,5 @@
+config = {
+    # The frontend should have these turned off in perpetuity.
+    'mbsconsumer': False,
+    'mbspoller': False,
+}

roles/mbs/frontend/files/mbs.wsgi

@@ -0,0 +1,6 @@
+#-*- coding: utf-8 -*-
+
+import logging
+logging.basicConfig(level='DEBUG')
+
+from module_build_service import app as application

roles/mbs/frontend/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+# Configuration for the Module Build Service (MBS) frontend webapp.
+
+- name: disable the scheduler on the frontend
+  copy: >
+    src={{ item }} dest=/etc/fedmsg.d/{{ item }}
+    owner=apache group=apache mode=0600
+  with_items:
+  - mbs-scheduler.py
+  notify:
+  - restart apache
+  tags:
+  - mbs
+  - mbs/frontend
+
+- name: copy mbs httpd config
+  template: >
+    src=mbs.conf dest=/etc/httpd/conf.d/mbs.conf
+    owner=apache group=apache mode=0644
+  notify:
+  - restart apache
+  tags:
+  - mbs
+  - mbs/frontend
+
+- name: copy custom wsgi file
+  copy: src=mbs.wsgi dest=/usr/share/mbs/mbs.wsgi mode=0644
+  notify:
+  - restart apache
+  tags:
+  - mbs
+  - mbs/frontend
+
+- name: ensure selinux lets httpd talk to postgres, memcached, and mail
+  seboolean: name={{item}} state=yes persistent=yes
+  with_items:
+  - httpd_can_network_connect_db
+  - httpd_can_network_memcache
+  - httpd_can_sendmail
+  tags:
+  - mbs
+  - mbs/frontend
+  - selinux

roles/mbs/frontend/templates/mbs.web.conf

@@ -0,0 +1,12 @@
+WSGIDaemonProcess mbs user=fedmsg group=fedmsg maximum-requests=1000 display-name=mbs processes={{ wsgi_procs }} threads={{ wsgi_threads }}
+WSGISocketPrefix run/wsgi
+WSGIRestrictStdout On
+WSGIRestrictSignal Off
+WSGIPythonOptimize 1
+
+WSGIScriptAlias /mbs /usr/share/mbs/mbs.wsgi
+
+<Location /notifications>
+    WSGIProcessGroup mbs
+    Require all granted
+</Location>