freshmaker: add freshmaker playbook and roles

This commit is contained in:
Qixiang Wan 2018-01-29 20:35:50 +08:00
parent 69ed1bb554
commit ffa7656480
12 changed files with 512 additions and 3 deletions

View file

@ -46,7 +46,7 @@
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: Set up apache on the frontend MBS API app
- name: set up Freshmaker frontend
hosts: freshmaker-frontend:freshmaker-frontend-stg
user: root
gather_facts: True
@ -58,12 +58,13 @@
roles:
- mod_wsgi
- freshmaker/frontend
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"
- name: set up fedmsg configuration and common freshmaker files
hosts: freshmaker:freshmaker-stg
- name: set up Freshmaker backend
hosts: freshmaker-backend:freshmaker-backend-stg
user: root
gather_facts: True
@ -74,6 +75,7 @@
roles:
- fedmsg/base
- freshmaker/backend
handlers:
- import_tasks: "{{ handlers_path }}/restart_services.yml"

View file

@ -0,0 +1,3 @@
---
freshmaker_upgrade: False
freshmaker_migrate_db: False

View file

@ -0,0 +1,3 @@
---
dependencies:
- { role: freshmaker/base }

View file

@ -0,0 +1,47 @@
---
- name: install the latest Freshmaker package
yum:
name: freshmaker
state: latest
update_cache: yes
with_items:
- freshmaker
- python2-odcs-client
when: freshmaker_upgrade
notify:
- restart fedmsg-hub
tags:
- freshmaker
- freshmaker/backend
- name: generate the Freshmaker koji config
template:
src: etc/koji.conf.d/freshmaker.conf.j2
dest: /etc/koji.conf.d/freshmaker.conf
owner: fedmsg
group: fedmsg
mode: 0440
notify:
- restart fedmsg-hub
tags:
- freshmaker
- freshmaker/backend
- name: ensure fedmsg-hub starts on boot
service:
name: "fedmsg-hub"
enabled: yes
# This will initialize Alembic if the database is empty, and migrate to the
# latest revision
- name: migrate the database
command: "{{ item }}"
with_items:
- freshmaker-manager upgradedb
- freshmaker-manager db migrate
become: yes
become_user: fedmsg
when: freshmaker_migrate_db
tags:
- freshmaker
- freshmaker/backend

View file

@ -0,0 +1,13 @@
[freshmaker_prod]
server = https://koji.fedoraproject.org/kojihub
weburl = https://koji.fedoraproject.org/koji
topurl = https://kojipkgs.fedoraproject.org/
authtype = kerberos
krb_rdns = false
[freshmaker_stg]
server = https://koji.stg.fedoraproject.org/kojihub
weburl = https://koji.stg.fedoraproject.org/koji
topurl = https://kojipkgs.stg.fedoraproject.org/
authtype = kerberos
krb_rdns = false

View file

@ -0,0 +1,27 @@
---
freshmaker_force_postgres_ssl: False
freshmaker_handler_build_whitelist: null
freshmaker_handler_build_blacklist: null
freshmaker_pdc_insecure: False
freshmaker_stg_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.stg.fedoraproject.org.keytab"
freshmaker_stg_krb_auth_principal: "freshmaker/freshmaker.stg.fedoraproject.org@STG.FEDORAPROJECT.ORG"
freshmaker_prod_krb_auth_client_keytab: "/etc/krb5.freshmaker_freshmaker.fedoraproject.org.keytab"
freshmaker_prod_krb_auth_principal: "freshmaker/freshmaker.fedoraproject.org@FEDORAPROJECT.ORG"
freshmaker_stg_git_base_url: git://pkgs.stg.fedoraproject.org
freshmaker_stg_git_ssh_base_url: ssh://%s@pkgs.stg.fedoraproject.org
freshmaker_stg_git_user: null
freshmaker_prod_git_base_url: git://pkgs.fedoraproject.org
freshmaker_prod_git_ssh_base_url: ssh://%s@pkgs.fedoraproject.org
freshmaker_prod_git_user: null
freshmaker_stg_odcs_server_url: https://odcs.fedoraproject.org
freshmaker_prod_odcs_server_url: https://odcs.stg.fedoraproject.org
freshmaker_stg_odcs_sigkeys: []
freshmaker_prod_odcs_sigkeys: []
freshmaker_dry_run: False
freshmaker_admins: {"users": [], "groups": []}
freshmaker_log_level: info
freshmaker_servername: localhost
freshmaker_messaging_topic_prefix: []
freshmaker_parsers: []
freshmaker_handlers: []

View file

@ -0,0 +1,52 @@
---
- name: install the packages required for Freshmaker frontend
yum:
name: "{{ item }}"
state: present
with_items:
- httpd
- mod_wsgi
- mod_auth_openidc
- libsemanage-python
- python-psycopg2
- freshmaker
when: inventory_hostname.startswith('freshmaker-frontend')
tags:
- freshmaker
- name: install the packages required for Freshmaker backend
yum:
name: "{{ item }}"
state: present
with_items:
- python-psycopg2
- freshmaker
when: inventory_hostname.startswith('freshmaker-backend')
tags:
- freshmaker
- name: generate Freshmaker app config for frontend
template:
src: etc/freshmaker/config.py.j2
dest: /etc/freshmaker/config.py
owner: apache
group: apache
mode: 0440
notify:
- restart apache
when: inventory_hostname.startswith('freshmaker-frontend')
tags:
- freshmaker
- name: generate Freshmaker app config for backend
template:
src: etc/freshmaker/config.py.j2
dest: /etc/freshmaker/config.py
owner: fedmsg
group: fedmsg
mode: 0440
notify:
- restart fedmsg-hub
when: inventory_hostname.startswith('freshmaker-backend')
tags:
- freshmaker

View file

@ -0,0 +1,255 @@
# -*- coding: utf-8 -*-
from os import path, environ
confdir = path.abspath(path.dirname(__file__))
# use parent dir as dbdir else fallback to current dir
dbdir = path.abspath(path.join(confdir, '..')) if confdir.endswith('conf') \
else confdir
class BaseConfiguration(object):
# Make this random (used to generate session keys)
SECRET_KEY = '74d9e9f9cd40e66fc6c4c2e9987dce48df3ce98542529fd0'
SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(path.join(
dbdir, 'freshmaker.db'))
SQLALCHEMY_TRACK_MODIFICATIONS = False
HOST = '0.0.0.0'
PORT = 5001
DEBUG = False
# Global network-related values, in seconds
NET_TIMEOUT = 120
NET_RETRY_INTERVAL = 30
SYSTEM = 'koji'
MESSAGING = 'fedmsg' # or amq
# Available backends are: console, file, journal.
LOG_BACKEND = 'journal'
# Path to log file when LOG_BACKEND is set to "file".
LOG_FILE = 'freshmaker.log'
# Available log levels are: debug, info, warn, error.
LOG_LEVEL = 'info'
MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.prod']
# Parsers defined for parse specific messages
PARSERS = [
'freshmaker.parsers.bodhi:BodhiUpdateCompleteStableParser',
'freshmaker.parsers.git:GitReceiveParser',
'freshmaker.parsers.koji:KojiTaskStateChangeParser',
'freshmaker.parsers.mbs:MBSModuleStateChangeParser',
]
# List of enabled composing handlers.
HANDLERS = [
"freshmaker.handlers.bodhi:BodhiUpdateCompleteStableHandler",
"freshmaker.handlers.git:GitDockerfileChangeHandler",
"freshmaker.handlers.git:GitModuleMetadataChangeHandler",
"freshmaker.handlers.git:GitRPMSpecChangeHandler",
"freshmaker.handlers.koji:KojiTaskStateChangeHandler",
"freshmaker.handlers.mbs:MBSModuleStateChangeHandler",
]
# Base URL of git repository with source artifacts.
GIT_BASE_URL = "git://pkgs.fedoraproject.org"
# SSH base URL of git repository
GIT_SSH_BASE_URL = "ssh://%s@pkgs.fedoraproject.org/"
# GIT user for cloning and pushing repo
GIT_USER = ""
# PDC API URL
PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1'
# Read Koji configuration from profile instead of reading them from
# configuration file directly. For staging Koji, it is stg.
KOJI_PROFILE = 'koji'
KOJI_PROXYUSER = False
KOJI_BUILD_OWNER = 'freshmaker'
# Settings for docker image rebuild handler
KOJI_CONTAINER_SCRATCH_BUILD = False
SSL_ENABLED = False
# whitelist and blacklist for handlers to decide whether an artifact
# can be built.
#
# In format of:
#
# { <handler_name> :
# { <artifact_type>: <list_of_name_branch_dict> }
# }
#
# Here is an example of allowing MBSModuleStateChangeHandler to build
# any module that module name matches 'base-.*' but not:
# 1. module name matches 'base-test-module'
# or:
# 2. module from branch 'rawhide'
#
# HANDLER_BUILD_WHITELIST = {
# "MBSModuleStateChangeHandler": {
# "module": [
# {
# 'name': 'base-.*',
# },
# ],
# },
# }
# HANDLER_BUILD_BLACKLIST = {
# "MBSModuleStateChangeHandler": {
# "module": [
# {
# 'name': 'base-test-module',
# },
# {
# 'branch': 'rawhide',
# },
# ],
# },
# }
class DevConfiguration(BaseConfiguration):
DEBUG = True
LOG_BACKEND = 'console'
LOG_LEVEL = 'debug'
MESSAGING_TOPIC_PREFIX = ['org.fedoraproject.dev', 'org.fedoraproject.stg']
# Global network-related values, in seconds
NET_TIMEOUT = 5
NET_RETRY_INTERVAL = 1
KOJI_CONTAINER_SCRATCH_BUILD = True
LIGHTBLUE_VERIFY_SSL = False
class TestConfiguration(BaseConfiguration):
LOG_BACKEND = 'console'
LOG_LEVEL = 'debug'
DEBUG = True
SQLALCHEMY_DATABASE_URI = 'sqlite:///{0}'.format(
path.join(dbdir, 'tests', 'test_freshmaker.db'))
MESSAGING = 'in_memory'
PDC_URL = 'http://pdc.fedoraproject.org/rest_api/v1'
# Global network-related values, in seconds
NET_TIMEOUT = 3
NET_RETRY_INTERVAL = 1
MBS_AUTH_TOKEN = "testingtoken"
KOJI_CONTAINER_SCRATCH_BUILD = True
LIGHTBLUE_SERVER_URL = '' # replace with real dev server url
LIGHTBLUE_VERIFY_SSL = False
class ProdConfiguration(BaseConfiguration):
AUTH_BACKEND = 'openidc'
# use kerberos for talking to koji
KRB_AUTH_USE_KEYTAB = True
PDC_INSECURE = {{ freshmaker_pdc_insecure }}
# No auth is required by Freshmaker, read-only PDC accesss is enough.
PDC_DEVELOP = True
{% if env == 'staging' %}
SECRET_KEY = "{{ freshmaker_stg_secret_key }}"
AUTH_OPENIDC_USERINFO_URI = 'https://id.stg.fedoraproject.org/openidc/UserInfo'
SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_stg_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}'
KOJI_PROFILE = 'freshmaker_stg'
PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1'
GIT_BASE_URL = "{{ freshmaker_stg_git_base_url }}"
GIT_SSH_BASE_URL = "{{ freshmaker_stg_git_ssh_base_url }}"
GIT_USER = "{{ freshmaker_stg_git_user }}"
ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}"
ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }}
KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_stg_krb_auth_client_keytab }}"
KRB_AUTH_PRINCIPAL = "{{ freshmaker_stg_krb_auth_principal }}"
{% else %}
SECRET_KEY = "{{ freshmaker_prod_secret_key }}"
AUTH_OPENIDC_USERINFO_URI = "{{ freshmaker_prod_auth_openidc_userinfo_uri }}"
SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://freshmaker:{{freshmaker_prod_db_password}}@db-freshmaker/freshmaker{{ '?sslmode=require' if freshmaker_force_postgres_ssl else '' }}'
KOJI_PROFILE = "freshmaker_production"
PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1'
GIT_BASE_URL = "{{ freshmaker_prod_git_base_url }}"
GIT_SSH_BASE_URL = "{{ freshmaker_prod_git_ssh_base_url }}"
GIT_USER = "{{ freshmaker_prod_git_user }}"
ODCS_SERVER_URL = "{{ freshmaker_prod_odcs_server_url }}"
ODCS_SIGKEYS = {{ freshmaker_prod_odcs_sigkeys }}
KRB_AUTH_CLIENT_KEYTAB = "{{ freshmaker_prod_krb_auth_client_keytab }}"
KRB_AUTH_PRINCIPAL = "{{ freshmaker_prod_krb_auth_principal }}"
{% endif %}
# requests_kerberos module does not support setting keytab, but the krb5
# library checks the KRB5_CLIENT_KTNAME environment variable to set the
# path to keytab.
environ["KRB5_CLIENT_KTNAME"] = KRB_AUTH_CLIENT_KEYTAB
MESSAGING = 'fedmsg'
MESSAGING_SENDER = 'fedmsg'
MESSAGING_BACKENDS = {
'fedmsg': {
'SERVICE': 'freshmaker',
},
'in_memory': {
'SERVICE': 'freshmaker',
}
}
MESSAGING_TOPIC_PREFIX = [
{% for prefix in freshmaker_messaging_topic_prefix %}
'{{ prefix }}',
{% endfor %}
]
PARSERS = [
{% for parser in freshmaker_parsers %}
'{{ parser }}',
{% endfor %}
]
HANDLERS = [
{% for handler in freshmaker_handlers %}
'{{ handler }}',
{% endfor %}
]
{% if freshmaker_handler_build_whitelist %}
HANDLER_BUILD_WHITELIST = {{ freshmaker_handler_build_whitelist }}
{% endif %}
{% if freshmaker_handler_build_blacklist %}
HANDLER_BUILD_BLACKLIST = {{ freshmaker_handler_build_blacklist }}
{% endif %}
DRY_RUN = {{ freshmaker_dry_run }}
ADMINS = {{ freshmaker_admins }}
LOG_LEVEL = "{{ freshmaker_log_level }}"
SERVER_NAME = "{{ freshmaker_servername }}"

View file

@ -0,0 +1,8 @@
---
freshmaker_upgrade: False
freshmaker_migrate_db: False
freshmaker_force_ssl: True
freshmaker_endpoint: ''
freshmaker_allowed_named_hosts: []
freshmaker_allowed_hosts: []
freshmaker_servername: localhost

View file

@ -0,0 +1,3 @@
---
dependencies:
- { role: freshmaker/base }

View file

@ -0,0 +1,47 @@
---
- name: install the latest Freshmaker package
yum:
name: freshmaker
state: latest
update_cache: yes
with_items:
- freshmaker
when: freshmaker_upgrade
notify:
- restart apache
tags:
- freshmaker
- freshmaker/frontend
- name: ensure selinux lets httpd talk to postgres
seboolean: name={{item}} state=yes persistent=yes
with_items:
- httpd_can_network_connect_db
- httpd_can_network_connect
when: "'enabled' in ansible_selinux.status"
tags:
- freshmaker
- freshmaker/frontend
- selinux
- name: make httpd logs world readable
file:
name: /var/log/httpd
state: directory
mode: 0755
tags:
- freshmaker
- freshmaker/frontend
- name: generate the Freshmaker httpd config
template:
src: etc/httpd/conf.d/freshmaker.conf.j2
dest: /etc/httpd/conf.d/freshmaker.conf
owner: apache
group: apache
mode: 0440
notify:
- restart apache
tags:
- freshmaker
- freshmaker/frontend

View file

@ -0,0 +1,49 @@
{% if freshmaker_force_ssl %}
# Force SSL
RewriteEngine On
RewriteCond %{HTTPS} off
RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
{% endif %}
WSGIDaemonProcess freshmaker user=fedmsg group=fedmsg threads=5 home=/usr/share/freshmaker
WSGIScriptAlias /{{ freshmaker_endpoint }} /usr/share/freshmaker/freshmaker.wsgi
{% if freshmaker_servername != inventory_hostname %}
# Redirect from the hostname of this machine to user-visible hostname.
RewriteEngine On
<If "%{HTTP_HOST} == '{{ inventory_hostname }}'">
RewriteRule (.*) "%{REQUEST_SCHEME}://{{ freshmaker_servername }}%{REQUEST_URI}" [R,L]
</If>
{% endif %}
{% if env == 'staging' %}
OIDCOAuthClientID {{ freshmaker_stg_oidc_client_id }}
OIDCOAuthClientSecret {{ freshmaker_stg_oidc_client_secret }}
OIDCOAuthIntrospectionEndpoint https://id.stg.fedoraproject.org/openidc/TokenInfo
{% else %}
OIDCOAuthClientID {{ freshmaker_prod_oidc_client_id }}
OIDCOAuthClientSecret {{ freshmaker_prod_oidc_client_secret }}
OIDCOAuthIntrospectionEndpoint https://id.fedoraproject.org/openidc/TokenInfo
{% endif %}
OIDCOAuthIntrospectionEndpointAuth client_secret_post
OIDCOAuthIntrospectionEndpointParams token_type_hint=Bearer
<Directory /usr/share/freshmaker>
WSGIProcessGroup freshmaker
WSGIApplicationGroup %{GLOBAL}
{% if freshmaker_allowed_named_hosts or freshmaker_allowed_hosts %}
# Only requests from following hosts/ips are allowed.
<RequireAny>
{{ 'Require host ' ~ freshmaker_allowed_named_hosts|join(' ') if freshmaker_allowed_named_hosts else '' }}
{{ 'Require ip ' ~ freshmaker_allowed_hosts|join(' ') if freshmaker_allowed_hosts else '' }}
</RequireAny>
{% endif %}
{% if not freshmaker_allowed_named_hosts and not freshmaker_allowed_hosts %}
# No auth mechanism configured, so everyone is allowed to access Freshmaker.
Require all granted
{% endif %}
</Directory>