From ffa7656480af958c79b3743263bcce269a04fe80 Mon Sep 17 00:00:00 2001 From: Qixiang Wan Date: Mon, 29 Jan 2018 20:35:50 +0800 Subject: [PATCH] freshmaker: add freshmaker playbook and roles --- playbooks/groups/freshmaker.yml | 8 +- roles/freshmaker/backend/defaults/main.yml | 3 + roles/freshmaker/backend/meta/main.yml | 3 + roles/freshmaker/backend/tasks/main.yml | 47 ++++ .../etc/koji.conf.d/freshmaker.conf.j2 | 13 + roles/freshmaker/base/defaults/main.yml | 27 ++ roles/freshmaker/base/tasks/main.yml | 52 ++++ .../templates/etc/freshmaker/config.py.j2 | 255 ++++++++++++++++++ roles/freshmaker/frontend/defaults/main.yml | 8 + roles/freshmaker/frontend/meta/main.yml | 3 + roles/freshmaker/frontend/tasks/main.yml | 47 ++++ .../etc/httpd/conf.d/freshmaker.conf.j2 | 49 ++++ 12 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 roles/freshmaker/backend/defaults/main.yml create mode 100644 roles/freshmaker/backend/meta/main.yml create mode 100644 roles/freshmaker/backend/tasks/main.yml create mode 100644 roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 create mode 100644 roles/freshmaker/base/defaults/main.yml create mode 100644 roles/freshmaker/base/tasks/main.yml create mode 100644 roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 create mode 100644 roles/freshmaker/frontend/defaults/main.yml create mode 100644 roles/freshmaker/frontend/meta/main.yml create mode 100644 roles/freshmaker/frontend/tasks/main.yml create mode 100644 roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 diff --git a/playbooks/groups/freshmaker.yml b/playbooks/groups/freshmaker.yml index 37677c14f8..968eebc5a6 100644 --- a/playbooks/groups/freshmaker.yml +++ b/playbooks/groups/freshmaker.yml @@ -46,7 +46,7 @@ handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" -- name: Set up apache on the frontend MBS API app +- name: set up Freshmaker frontend hosts: freshmaker-frontend:freshmaker-frontend-stg user: root gather_facts: True @@ -58,12 +58,13 @@ roles: - mod_wsgi + - freshmaker/frontend handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" -- name: set up fedmsg configuration and common freshmaker files - hosts: freshmaker:freshmaker-stg +- name: set up Freshmaker backend + hosts: freshmaker-backend:freshmaker-backend-stg user: root gather_facts: True @@ -74,6 +75,7 @@ roles: - fedmsg/base + - freshmaker/backend handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/freshmaker/backend/defaults/main.yml b/roles/freshmaker/backend/defaults/main.yml new file mode 100644 index 0000000000..1adc701cc4 --- /dev/null +++ b/roles/freshmaker/backend/defaults/main.yml @@ -0,0 +1,3 @@ +--- +freshmaker_upgrade: False +freshmaker_migrate_db: False diff --git a/roles/freshmaker/backend/meta/main.yml b/roles/freshmaker/backend/meta/main.yml new file mode 100644 index 0000000000..4a5c132f49 --- /dev/null +++ b/roles/freshmaker/backend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: freshmaker/base } diff --git a/roles/freshmaker/backend/tasks/main.yml b/roles/freshmaker/backend/tasks/main.yml new file mode 100644 index 0000000000..febb901756 --- /dev/null +++ b/roles/freshmaker/backend/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: install the latest Freshmaker package + yum: + name: freshmaker + state: latest + update_cache: yes + with_items: + - freshmaker + - python2-odcs-client + when: freshmaker_upgrade + notify: + - restart fedmsg-hub + tags: + - freshmaker + - freshmaker/backend + +- name: generate the Freshmaker koji config + template: + src: etc/koji.conf.d/freshmaker.conf.j2 + dest: /etc/koji.conf.d/freshmaker.conf + owner: fedmsg + group: fedmsg + mode: 0440 + notify: + - restart fedmsg-hub + tags: + - freshmaker + - freshmaker/backend + +- name: ensure fedmsg-hub starts on boot + service: + name: "fedmsg-hub" + enabled: yes + +# This will initialize Alembic if the database is empty, and migrate to the +# latest revision +- name: migrate the database + command: "{{ item }}" + with_items: + - freshmaker-manager upgradedb + - freshmaker-manager db migrate + become: yes + become_user: fedmsg + when: freshmaker_migrate_db + tags: + - freshmaker + - freshmaker/backend diff --git a/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 b/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 new file mode 100644 index 0000000000..ed2dcc8653 --- /dev/null +++ b/roles/freshmaker/backend/templates/etc/koji.conf.d/freshmaker.conf.j2 @@ -0,0 +1,13 @@ +[freshmaker_prod] +server = https://koji.fedoraproject.org/kojihub +weburl = https://koji.fedoraproject.org/koji +topurl = https://kojipkgs.fedoraproject.org/ +authtype = kerberos +krb_rdns = false + +[freshmaker_stg] +server = https://koji.stg.fedoraproject.org/kojihub +weburl = https://koji.stg.fedoraproject.org/koji +topurl = https://kojipkgs.stg.fedoraproject.org/ +authtype = kerberos +krb_rdns = false diff --git a/roles/freshmaker/base/defaults/main.yml b/roles/freshmaker/base/defaults/main.yml new file mode 100644 index 0000000000..44d59bbea4 --- /dev/null +++ b/roles/freshmaker/base/defaults/main.yml @@ -0,0 +1,27 @@ +--- +freshmaker_force_postgres_ssl: False +freshmaker_handler_build_whitelist: null +freshmaker_handler_build_blacklist: null +freshmaker_pdc_insecure: False +freshmaker_stg_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.stg.fedoraproject.org.keytab" +freshmaker_stg_krb_auth_principal: "freshmaker/freshmaker.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG" +freshmaker_prod_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.fedoraproject.org.keytab" +freshmaker_prod_krb_auth_principal: "freshmaker/freshmaker.fedoraproject.org@FEDORAPROJECT.ORG" +freshmaker_stg_git_base_url: git://pkgs.stg.fedoraproject.org +freshmaker_stg_git_ssh_base_url: ssh://%s@pkgs.stg.fedoraproject.org +freshmaker_stg_git_user: null +freshmaker_prod_git_base_url: git://pkgs.fedoraproject.org +freshmaker_prod_git_ssh_base_url: ssh://%s@pkgs.fedoraproject.org +freshmaker_prod_git_user: null +freshmaker_stg_odcs_server_url: https://odcs.fedoraproject.org +freshmaker_prod_odcs_server_url: https://odcs.stg.fedoraproject.org +freshmaker_stg_odcs_sigkeys: [] +freshmaker_prod_odcs_sigkeys: [] +freshmaker_dry_run: False +freshmaker_admins: {"users": [], "groups": []} +freshmaker_log_level: info +freshmaker_servername: localhost + +freshmaker_messaging_topic_prefix: [] +freshmaker_parsers: [] +freshmaker_handlers: [] diff --git a/roles/freshmaker/base/tasks/main.yml b/roles/freshmaker/base/tasks/main.yml new file mode 100644 index 0000000000..2922e77483 --- /dev/null +++ b/roles/freshmaker/base/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: install the packages required for Freshmaker frontend + yum: + name: "{{ item }}" + state: present + with_items: + - httpd + - mod_wsgi + - mod_auth_openidc + - libsemanage-python + - python-psycopg2 + - freshmaker + when: inventory_hostname.startswith('freshmaker-frontend') + tags: + - freshmaker + +- name: install the packages required for Freshmaker backend + yum: + name: "{{ item }}" + state: present + with_items: + - python-psycopg2 + - freshmaker + when: inventory_hostname.startswith('freshmaker-backend') + tags: + - freshmaker + +- name: generate Freshmaker app config for frontend + template: + src: etc/freshmaker/config.py.j2 + dest: /etc/freshmaker/config.py + owner: apache + group: apache + mode: 0440 + notify: + - restart apache + when: inventory_hostname.startswith('freshmaker-frontend') + tags: + - freshmaker + +- name: generate Freshmaker app config for backend + template: + src: etc/freshmaker/config.py.j2 + dest: /etc/freshmaker/config.py + owner: fedmsg + group: fedmsg + mode: 0440 + notify: + - restart fedmsg-hub + when: inventory_hostname.startswith('freshmaker-backend') + tags: + - freshmaker diff --git a/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 new file mode 100644 index 0000000000..8a03a4aaa5 --- /dev/null +++ b/roles/freshmaker/base/templates/etc/freshmaker/config.py.j2 @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +from os import path, environ + +confdir = path.abspath(path.dirname(__file__)) +# use parent dir as dbdir else fallback to current dir +dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \ + else confdir + + +class BaseConfiguration(object): + # Make this random (used to generate session keys) + SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0' + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join( + dbdir, 'freshmaker.db')) + SQLALCHEMY_TRACK_MODIFICATIONS = False + + HOST = '0.0.0.0' + PORT = 5001 + + DEBUG = False + # Global network-related values, in seconds + NET_TIMEOUT = 120 + NET_RETRY_INTERVAL = 30 + + SYSTEM = 'koji' + MESSAGING = 'fedmsg' # or amq + + # Available backends are: console, file, journal. + LOG_BACKEND = 'journal' + + # Path to log file when LOG_BACKEND is set to "file". + LOG_FILE = 'freshmaker.log' + + # Available log levels are: debug, info, warn, error. + LOG_LEVEL = 'info' + + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod'] + + # Parsers defined for parse specific messages + PARSERS = [ + 'freshmaker.parsers.bodhi:BodhiUpdateCompleteStableParser', + 'freshmaker.parsers.git:GitReceiveParser', + 'freshmaker.parsers.koji:KojiTaskStateChangeParser', + 'freshmaker.parsers.mbs:MBSModuleStateChangeParser', + ] + + # List of enabled composing handlers. + HANDLERS = [ + "freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler", + "freshmaker.handlers.git:GitDockerfileChangeHandler", + "freshmaker.handlers.git:GitModuleMetadataChangeHandler", + "freshmaker.handlers.git:GitRPMSpecChangeHandler", + "freshmaker.handlers.koji:KojiTaskStateChangeHandler", + "freshmaker.handlers.mbs:MBSModuleStateChangeHandler", + ] + + # Base URL of git repository with source artifacts. + GIT_BASE_URL = "git://pkgs.fedoraproject.org" + + # SSH base URL of git repository + GIT_SSH_BASE_URL = "ssh://%s@pkgs.fedoraproject.org/" + + # GIT user for cloning and pushing repo + GIT_USER = "" + + # PDC API URL + PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + + # Read Koji configuration from profile instead of reading them from + # configuration file directly. For staging Koji, it is stg. + KOJI_PROFILE = 'koji' + KOJI_PROXYUSER = False + KOJI_BUILD_OWNER = 'freshmaker' + + # Settings for docker image rebuild handler + KOJI_CONTAINER_SCRATCH_BUILD = False + + SSL_ENABLED = False + + # whitelist and blacklist for handlers to decide whether an artifact + # can be built. + # + # In format of: + # + # { : + # { : } + # } + # + # Here is an example of allowing MBSModuleStateChangeHandler to build + # any module that module name matches 'base-.*' but not: + # 1. module name matches 'base-test-module' + # or: + # 2. module from branch 'rawhide' + # + # HANDLER_BUILD_WHITELIST = { + # "MBSModuleStateChangeHandler": { + # "module": [ + # { + # 'name': 'base-.*', + # }, + # ], + # }, + # } + # HANDLER_BUILD_BLACKLIST = { + # "MBSModuleStateChangeHandler": { + # "module": [ + # { + # 'name': 'base-test-module', + # }, + # { + # 'branch': 'rawhide', + # }, + # ], + # }, + # } + + +class DevConfiguration(BaseConfiguration): + DEBUG = True + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + + MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.dev', 'org.fedoraproject.stg'] + + # Global network-related values, in seconds + NET_TIMEOUT = 5 + NET_RETRY_INTERVAL = 1 + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_VERIFY_SSL = False + + +class TestConfiguration(BaseConfiguration): + LOG_BACKEND = 'console' + LOG_LEVEL = 'debug' + DEBUG = True + + SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format( + path.join(dbdir, 'tests', 'test_freshmaker.db')) + + MESSAGING = 'in_memory' + PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1' + + # Global network-related values, in seconds + NET_TIMEOUT = 3 + NET_RETRY_INTERVAL = 1 + MBS_AUTH_TOKEN = "testingtoken" + + KOJI_CONTAINER_SCRATCH_BUILD = True + + LIGHTBLUE_SERVER_URL = '' # replace with real dev server url + LIGHTBLUE_VERIFY_SSL = False + + +class ProdConfiguration(BaseConfiguration): + AUTH_BACKEND = 'openidc' + # use kerberos for talking to koji + KRB_AUTH_USE_KEYTAB = True + + PDC_INSECURE = {{ freshmaker_pdc_insecure }} + # No auth is required by Freshmaker, read-only PDC accesss is enough. + PDC_DEVELOP = True + +{% if env == 'staging' %} + SECRET_KEY = "{{ freshmaker_stg_secret_key }}" + + AUTH_OPENIDC_USERINFO_URI = 'https://id.stg.fedoraproject.org/openidc/UserInfo' + + SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_stg_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}' + + KOJI_PROFILE = 'freshmaker_stg' + + PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1' + + GIT_BASE_URL = "{{ freshmaker_stg_git_base_url }}" + GIT_SSH_BASE_URL = "{{ freshmaker_stg_git_ssh_base_url }}" + GIT_USER = "{{ freshmaker_stg_git_user }}" + + ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}" + ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }} + + KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_stg_krb_auth_client_keytab }}" + KRB_AUTH_PRINCIPAL = "{{ freshmaker_stg_krb_auth_principal }}" +{% else %} + SECRET_KEY = "{{ freshmaker_prod_secret_key }}" + + AUTH_OPENIDC_USERINFO_URI = "{{ freshmaker_prod_auth_openidc_userinfo_uri }}" + + SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_prod_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}' + + KOJI_PROFILE = "freshmaker_production" + + PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1' + + GIT_BASE_URL = "{{ freshmaker_prod_git_base_url }}" + GIT_SSH_BASE_URL = "{{ freshmaker_prod_git_ssh_base_url }}" + GIT_USER = "{{ freshmaker_prod_git_user }}" + + ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}" + ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }} + + KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_prod_krb_auth_client_keytab }}" + KRB_AUTH_PRINCIPAL = "{{ freshmaker_prod_krb_auth_principal }}" +{% endif %} + + # requests_kerberos module does not support setting keytab, but the krb5 + # library checks the KRB5_CLIENT_KTNAME environment variable to set the + # path to keytab. + environ["KRB5_CLIENT_KTNAME"] = KRB_AUTH_CLIENT_KEYTAB + + MESSAGING = 'fedmsg' + MESSAGING_SENDER = 'fedmsg' + MESSAGING_BACKENDS = { + 'fedmsg': { + 'SERVICE': 'freshmaker', + }, + 'in_memory': { + 'SERVICE': 'freshmaker', + } + } + + MESSAGING_TOPIC_PREFIX = [ + {% for prefix in freshmaker_messaging_topic_prefix %} + '{{ prefix }}', + {% endfor %} + ] + + PARSERS = [ + {% for parser in freshmaker_parsers %} + '{{ parser }}', + {% endfor %} + ] + + HANDLERS = [ + {% for handler in freshmaker_handlers %} + '{{ handler }}', + {% endfor %} + ] + +{% if freshmaker_handler_build_whitelist %} + HANDLER_BUILD_WHITELIST = {{ freshmaker_handler_build_whitelist }} +{% endif %} + +{% if freshmaker_handler_build_blacklist %} + HANDLER_BUILD_BLACKLIST = {{ freshmaker_handler_build_blacklist }} +{% endif %} + + DRY_RUN = {{ freshmaker_dry_run }} + + ADMINS = {{ freshmaker_admins }} + + LOG_LEVEL = "{{ freshmaker_log_level }}" + SERVER_NAME = "{{ freshmaker_servername }}" diff --git a/roles/freshmaker/frontend/defaults/main.yml b/roles/freshmaker/frontend/defaults/main.yml new file mode 100644 index 0000000000..2513ba4fc6 --- /dev/null +++ b/roles/freshmaker/frontend/defaults/main.yml @@ -0,0 +1,8 @@ +--- +freshmaker_upgrade: False +freshmaker_migrate_db: False +freshmaker_force_ssl: True +freshmaker_endpoint: '' +freshmaker_allowed_named_hosts: [] +freshmaker_allowed_hosts: [] +freshmaker_servername: localhost diff --git a/roles/freshmaker/frontend/meta/main.yml b/roles/freshmaker/frontend/meta/main.yml new file mode 100644 index 0000000000..4a5c132f49 --- /dev/null +++ b/roles/freshmaker/frontend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - { role: freshmaker/base } diff --git a/roles/freshmaker/frontend/tasks/main.yml b/roles/freshmaker/frontend/tasks/main.yml new file mode 100644 index 0000000000..8da7c575eb --- /dev/null +++ b/roles/freshmaker/frontend/tasks/main.yml @@ -0,0 +1,47 @@ +--- +- name: install the latest Freshmaker package + yum: + name: freshmaker + state: latest + update_cache: yes + with_items: + - freshmaker + when: freshmaker_upgrade + notify: + - restart apache + tags: + - freshmaker + - freshmaker/frontend + +- name: ensure selinux lets httpd talk to postgres + seboolean: name={{item}} state=yes persistent=yes + with_items: + - httpd_can_network_connect_db + - httpd_can_network_connect + when: "'enabled' in ansible_selinux.status" + tags: + - freshmaker + - freshmaker/frontend + - selinux + +- name: make httpd logs world readable + file: + name: /var/log/httpd + state: directory + mode: 0755 + tags: + - freshmaker + - freshmaker/frontend + +- name: generate the Freshmaker httpd config + template: + src: etc/httpd/conf.d/freshmaker.conf.j2 + dest: /etc/httpd/conf.d/freshmaker.conf + owner: apache + group: apache + mode: 0440 + notify: + - restart apache + tags: + - freshmaker + - freshmaker/frontend diff --git a/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 new file mode 100644 index 0000000000..213002c28e --- /dev/null +++ b/roles/freshmaker/frontend/templates/etc/httpd/conf.d/freshmaker.conf.j2 @@ -0,0 +1,49 @@ +{% if freshmaker_force_ssl %} +# Force SSL +RewriteEngine On +RewriteCond %{HTTPS} off +RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} +{% endif %} + +WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg threads=5 home=/usr/share/freshmaker +WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi + +{% if freshmaker_servername != inventory_hostname %} +# Redirect from the hostname of this machine to user-visible hostname. +RewriteEngine On + +RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L] + +{% endif %} + +{% if env == 'staging' %} +OIDCOAuthClientID {{ freshmaker_stg_oidc_client_id }} +OIDCOAuthClientSecret {{ freshmaker_stg_oidc_client_secret }} +OIDCOAuthIntrospectionEndpoint https://id.stg.fedoraproject.org/openidc/TokenInfo +{% else %} +OIDCOAuthClientID {{ freshmaker_prod_oidc_client_id }} +OIDCOAuthClientSecret {{ freshmaker_prod_oidc_client_secret }} +OIDCOAuthIntrospectionEndpoint https://id.fedoraproject.org/openidc/TokenInfo +{% endif %} + +OIDCOAuthIntrospectionEndpointAuth client_secret_post +OIDCOAuthIntrospectionEndpointParams token_type_hint=Bearer + + + WSGIProcessGroup freshmaker + WSGIApplicationGroup %{GLOBAL} + + {% if freshmaker_allowed_named_hosts or freshmaker_allowed_hosts %} + # Only requests from following hosts/ips are allowed. + + {{ 'Require host ' ~ freshmaker_allowed_named_hosts|join(' ') if freshmaker_allowed_named_hosts else '' }} + {{ 'Require ip ' ~ freshmaker_allowed_hosts|join(' ') if freshmaker_allowed_hosts else '' }} + + {% endif %} + + {% if not freshmaker_allowed_named_hosts and not freshmaker_allowed_hosts %} + # No auth mechanism configured, so everyone is allowed to access Freshmaker. + Require all granted + {% endif %} + +