From 1e4618fcc5082d53adf341879c11859d18108813 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 30 Jan 2017 16:59:33 +0000 Subject: [PATCH] First draft of the mbs/frontend role. --- playbooks/groups/mbs.yml | 9 +- roles/mbs/common/tasks/main.yml | 27 +++++ roles/mbs/common/templates/config.py | 119 ++++++++++++++++++++++ roles/mbs/common/templates/koji.conf | 13 +++ roles/mbs/frontend/files/mbs-scheduler.py | 5 + roles/mbs/frontend/files/mbs.wsgi | 6 ++ roles/mbs/frontend/tasks/main.yml | 43 ++++++++ roles/mbs/frontend/templates/mbs.web.conf | 12 +++ 8 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 roles/mbs/common/tasks/main.yml create mode 100644 roles/mbs/common/templates/config.py create mode 100644 roles/mbs/common/templates/koji.conf create mode 100644 roles/mbs/frontend/files/mbs-scheduler.py create mode 100644 roles/mbs/frontend/files/mbs.wsgi create mode 100644 roles/mbs/frontend/tasks/main.yml create mode 100644 roles/mbs/frontend/templates/mbs.web.conf diff --git a/playbooks/groups/mbs.yml b/playbooks/groups/mbs.yml index bbca9a55ba..ada649a9f6 100644 --- a/playbooks/groups/mbs.yml +++ b/playbooks/groups/mbs.yml @@ -62,7 +62,7 @@ handlers: - include: "{{ handlers }}/restart_services.yml" -- name: set up fedmsg configuration +- name: set up fedmsg configuration and common mbs files hosts: mbs:mbs-stg user: root gather_facts: True @@ -74,6 +74,7 @@ roles: - fedmsg/base + - mbs/common handlers: - include: "{{ handlers }}/restart_services.yml" @@ -89,7 +90,7 @@ - "{{ vars_path }}/{{ ansible_distribution }}.yml" roles: - #- mbs/frontend + - mbs/frontend handlers: - include: "{{ handlers }}/restart_services.yml" @@ -105,6 +106,10 @@ - "{{ vars_path }}/{{ ansible_distribution }}.yml" roles: + - role: keytab/service + service: mbs + owner_user: fedmsg + host: "mbs{{env_suffix}}.fedoraproject.org" - fedmsg/hub #- mbs/backend diff --git a/roles/mbs/common/tasks/main.yml b/roles/mbs/common/tasks/main.yml new file mode 100644 index 0000000000..32280c3ad9 --- /dev/null +++ b/roles/mbs/common/tasks/main.yml @@ -0,0 +1,27 @@ +--- +# Common configuration for the Module Build Service (MBS) pieces + +- name: install needed packages + yum: pkg={{ item }} state=present + with_items: + - module-build-service + - python-psycopg2 + - libsemanage-python + - python-memcached + notify: + - restart apache + - restart fedmsg-hub + tags: + - mbs + - mbs/common + +- name: copy app configuration + template: > + src=config.py dest=/etc/module-build-service/config.py + owner=root group=fedmsg mode=0640 + notify: + - restart apache + - restart fedmsg-hub + tags: + - mbs + - mbs/common diff --git a/roles/mbs/common/templates/config.py b/roles/mbs/common/templates/config.py new file mode 100644 index 0000000000..0837a8bbcd --- /dev/null +++ b/roles/mbs/common/templates/config.py @@ -0,0 +1,119 @@ +from os import path + +# FIXME: workaround for this moment till confdir, dbdir (installdir etc.) are +# declared properly somewhere/somehow +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \ + else confdir + + +class BaseConfiguration(object): + DEBUG = False + # Make this random (used to generate session keys) + SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0' + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join( + dbdir, 'module_build_service.db')) + SQLALCHEMY_TRACK_MODIFICATIONS = True + # Where we should run when running "manage.py runssl" directly. + HOST = '0.0.0.0' + PORT = 5000 + + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = 'koji' + MESSAGING = 'fedmsg' # or amq + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] + KOJI_CONFIG = '/etc/module-build-service/koji.conf' + KOJI_PROFILE = 'koji' + KOJI_ARCHES = ['i686', 'armv7hl', 'x86_64'] + KOJI_PROXYUSER = True + KOJI_REPOSITORY_URL = 'https://kojipkgs.stg.fedoraproject.org/repos' + COPR_CONFIG = '/etc/module-build-service/copr.conf' + PDC_URL = 'http://modularity.fedorainfracloud.org:8080/rest_api/v1' + PDC_INSECURE = True + PDC_DEVELOP = True + SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"] + + # How often should we resort to polling, in seconds + # Set to zero to disable polling + POLLING_INTERVAL = 600 + + # Determines how many builds that can be submitted to the builder + # and be in the build state at a time. Set this to 0 for no restrictions + NUM_CONSECUTIVE_BUILDS = 5 + + RPMS_DEFAULT_REPOSITORY = 'git://pkgs.fedoraproject.org/rpms/' + RPMS_ALLOW_REPOSITORY = False + RPMS_DEFAULT_CACHE = 'http://pkgs.fedoraproject.org/repo/pkgs/' + RPMS_ALLOW_CACHE = False + + # Available backends are: console, file, journal. + LOG_BACKEND = 'journal' + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = 'module_build_service.log' + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = 'info' + + # Settings for Kerberos + KRB_KEYTAB = None + KRB_PRINCIPAL = None + KRB_CCACHE = None + + # AMQ prefixed variables are required only while using 'amq' as messaging backend + # Addresses to listen to + AMQ_RECV_ADDRESSES = ['amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.koji', + 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service'] + # Address for sending messages + AMQ_DEST_ADDRESS = 'amqps://messaging.mydomain.com/Consumer.m8y.VirtualTopic.eng.module_build_service' + AMQ_CERT_FILE = '/etc/module_build_service/msg-m8y-client.crt' + AMQ_PRIVATE_KEY_FILE = '/etc/module_build_service/msg-m8y-client.key' + AMQ_TRUSTED_CERT_FILE = '/etc/module_build_service/Root-CA.crt' + + +class ProdConfiguration(BaseConfiguration): + DEBUG = False # Don't turn this on. + + {% if env == 'staging' %} + SECRET_KEY = '{{ mbs_stg_secret_key }}' + SQLALCHEMY_DATABASE_URI = 'postgresql://mbs:{{mbs_stg_db_password}}@db-mbs/mbs' + {% else %} + SECRET_KEY = '{{ mbs_prod_secret_key }}' + SQLALCHEMY_DATABASE_URI = 'postgresql://mbs:{{mbs_prod_db_password}}@db-mbs/mbs' + {% endif %} + + {% if env == 'staging' %} + KRB_PRINCIPAL = 'modularity@STG.FEDORAPROJECT.ORG' + {% else %} + KRB_PRINCIPAL = 'modularity@FEDORAPROJECT.ORG' + {% endif %} + + KRB_KEYTAB = '/etc/krb5.mbs_mbs{{env_suffix}}.fedoraproject.org.keytab' + KRB_CCACHE = '/var/cache/mbs-krb5cc' + + LOG_LEVEL = 'debug' + LOG_BACKEND = 'console' + + PDC_INSECURE = False + PDC_DEVELOP = False + + KOJI_CONFIG = path.join(confdir, 'koji.conf') + {% if env == 'staging' %} + KOJI_PROFILE = 'staging' + KOJI_ARCHES = ['x86_64'] + KOJI_REPOSITORY_URL = 'http://kojipkgs.stg.fedoraproject.org/repos' + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.stg'] + PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1' + SCMURLS = ["git://pkgs.stg.fedoraproject.org/modules/"] + {% else %} + KOJI_PROFILE = 'production' + KOJI_ARCHES = ['x86_64'] + KOJI_REPOSITORY_URL = 'http://kojipkgs.fedoraproject.org/repos' + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' + SCMURLS = ["git://pkgs.fedoraproject.org/modules/"] + {% endif %} diff --git a/roles/mbs/common/templates/koji.conf b/roles/mbs/common/templates/koji.conf new file mode 100644 index 0000000000..89d092f3f6 --- /dev/null +++ b/roles/mbs/common/templates/koji.conf @@ -0,0 +1,13 @@ +[production] +server = https://koji.fedoraproject.org/kojihub +weburl = https://koji.fedoraproject.org/koji +topurl = https://kojipkgs.fedoraproject.org/ +authtype = kerberos +krb_rdns = false + +[staging] +server = https://koji.stg.fedoraproject.org/kojihub +weburl = https://koji.stg.fedoraproject.org/koji +topurl = https://kojipkgs.stg.fedoraproject.org/ +authtype = kerberos +krb_rdns = false diff --git a/roles/mbs/frontend/files/mbs-scheduler.py b/roles/mbs/frontend/files/mbs-scheduler.py new file mode 100644 index 0000000000..0bf66eb360 --- /dev/null +++ b/roles/mbs/frontend/files/mbs-scheduler.py @@ -0,0 +1,5 @@ +config = { + # The frontend should have these turned off in perpetuity. + 'mbsconsumer': False, + 'mbspoller': False, +} diff --git a/roles/mbs/frontend/files/mbs.wsgi b/roles/mbs/frontend/files/mbs.wsgi new file mode 100644 index 0000000000..e3c7bd5223 --- /dev/null +++ b/roles/mbs/frontend/files/mbs.wsgi @@ -0,0 +1,6 @@ +#-*- coding: utf-8 -*- + +import logging +logging.basicConfig(level='DEBUG') + +from module_build_service import app as application diff --git a/roles/mbs/frontend/tasks/main.yml b/roles/mbs/frontend/tasks/main.yml new file mode 100644 index 0000000000..e40654e4a9 --- /dev/null +++ b/roles/mbs/frontend/tasks/main.yml @@ -0,0 +1,43 @@ +--- +# Configuration for the Module Build Service (MBS) frontend webapp. + +- name: disable the scheduler on the frontend + copy: > + src={{ item }} dest=/etc/fedmsg.d/{{ item }} + owner=apache group=apache mode=0600 + with_items: + - mbs-scheduler.py + notify: + - restart apache + tags: + - mbs + - mbs/frontend + +- name: copy mbs httpd config + template: > + src=mbs.conf dest=/etc/httpd/conf.d/mbs.conf + owner=apache group=apache mode=0644 + notify: + - restart apache + tags: + - mbs + - mbs/frontend + +- name: copy custom wsgi file + copy: src=mbs.wsgi dest=/usr/share/mbs/mbs.wsgi mode=0644 + notify: + - restart apache + tags: + - mbs + - mbs/frontend + +- name: ensure selinux lets httpd talk to postgres, memcached, and mail + seboolean: name={{item}} state=yes persistent=yes + with_items: + - httpd_can_network_connect_db + - httpd_can_network_memcache + - httpd_can_sendmail + tags: + - mbs + - mbs/frontend + - selinux diff --git a/roles/mbs/frontend/templates/mbs.web.conf b/roles/mbs/frontend/templates/mbs.web.conf new file mode 100644 index 0000000000..1d6c3d2a60 --- /dev/null +++ b/roles/mbs/frontend/templates/mbs.web.conf @@ -0,0 +1,12 @@ +WSGIDaemonProcess mbs user=fedmsg group=fedmsg maximum-requests=1000 display-name=mbs processes={{ wsgi_procs }} threads={{ wsgi_threads }} +WSGISocketPrefix run/wsgi +WSGIRestrictStdout On +WSGIRestrictSignal Off +WSGIPythonOptimize 1 + +WSGIScriptAlias /mbs /usr/share/mbs/mbs.wsgi + + + WSGIProcessGroup mbs + Require all granted +