Vault deployment on dev

inventory/group_vars/resultsdb-dev

@@ -49,3 +49,17 @@ execdb_db_name: execdb_dev
 execdb_db_user: "{{ dev_execdb_db_user }}"
 execdb_db_password: "{{ dev_execdb_db_password }}"
 execdb_secret_key: "{{ dev_execdb_secret_key }}"
+
+
+############################################################
+# vault details
+############################################################
+vault_db_host_machine: db-qa01.qa.fedoraproject.org
+vault_db_host: "{{ vault_db_host_machine }}"
+vault_db_port: 5432
+vault_endpoint: 'vault'
+vault_db_name: vault_dev
+vault_db_user: "{{ dev_vault_db_user }}"
+vault_db_password: "{{ dev_vault_db_password }}"
+vault_secret_key: "{{ dev_vault_secret_key }}"
+vault_masterkey: "{{dev_vault_masterkey}}"

inventory/group_vars/taskotron-dev

@@ -37,6 +37,8 @@ execdb_endpoint: execdb
 execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbottest
 execdb_server: http://resultsdb-dev01.qa.fedoraproject.org/execdb
 
+vault_server: http://resultsdb-dev01.qa.fedoraproject.org/vault
+
 resultsdb_api_endpoint: resultsdb_api
 resultsdb_host: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb_api/
 resultsdb_external_url: https://taskotron-dev.fedoraproject.org/resultsdb/

inventory/host_vars/db-qa01.qa.fedoraproject.org

@@ -17,6 +17,7 @@ databases:
 - execdb
 - openqa
 - resultsdb
+- vault
 
 # This is a more strict list, to be made publicly available
 dbs_to_backup:
@@ -34,6 +35,7 @@ dbs_to_backup:
 - resultsdb
 - resultsdb_stg
 - resultsdb_dev
+- vault_dev
 
 mariadb_root_password: "{{ dbqa01_mysql_root_password }}"
 

playbooks/groups/resultsdb-dev.yml

@@ -37,7 +37,7 @@
   handlers:
    - import_tasks: "{{ handlers_path }}/restart_services.yml"
 
-- name: configure resultsdb and execdb
+- name: configure resultsdb, execdb and vault
   hosts: resultsdb-dev
   user: root
   gather_facts: True
@@ -51,6 +51,7 @@
    - { role: taskotron/resultsdb-backend, tags: ['resultsdb-be'] }
    - { role: taskotron/resultsdb-frontend, tags: ['resultsdb-fe'] }
    - { role: taskotron/execdb, tags: ['execdb'] }
+   - { role: taskotron/vault, tags: ['vault'] }
 
   handlers:
    - import_tasks: "{{ handlers_path }}/restart_services.yml"

playbooks/include/proxies-reverseproxy.yml

@@ -426,6 +426,15 @@
     # Talk directly to the app server, not haproxy
     proxyurl: http://resultsdb01.vpn.fedoraproject.org
 
+  - role: httpd/reverseproxy
+    website: taskotron.fedoraproject.org
+    destname: taskotron-vault
+    localpath: /vault
+    remotepath: /vault
+    # Talk directly to the app server, not haproxy
+    proxyurl: http://resultsdb01.vpn.fedoraproject.org
+
+
   ### And four entries for taskotron for staging
   - role: httpd/reverseproxy
     website: taskotron.stg.fedoraproject.org

roles/taskotron/taskotron-proxy/tasks/main.yml

@@ -30,3 +30,8 @@
   template: src=execdb.conf.j2 dest=/etc/httpd/conf.d/execdb.conf owner=root group=root
   notify:
     - reload httpd
+
+- name: copy vault proxy httpd config
+  template: src=vault.conf.j2 dest=/etc/httpd/conf.d/vault.conf owner=root group=root
+  notify:
+    - reload httpd

roles/taskotron/taskotron-proxy/templates/vault.conf.j2

@@ -0,0 +1,5 @@
+<Location /{{ vault_endpoint }}/ >
+  ProxyPass {{ vault_server }}/
+  ProxyPassReverse {{ vault_server }}/
+  RequestHeader add X-Script-Name /{{ vault_endpoint }}/
+</Location>

roles/taskotron/vault/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+extra_enablerepos: ''

roles/taskotron/vault/tasks/main.yml

@@ -0,0 +1,58 @@
+---
+- name: start httpd (provided in the apache role)
+  service: name=httpd state=started
+
+- name: ensure packages required for vault are installed (yum)
+  package: name={{ item }} state=present
+  with_items:
+    - vault
+    - mod_wsgi
+    - python-psycopg2
+    - libsemanage-python
+  when: ansible_distribution_major_version|int < 22
+
+- name: ensure packages required for vault are installed (dnf)
+  dnf: name={{ item }} state=present enablerepo={{ extra_enablerepos }}
+  with_items:
+    - vault
+    - mod_wsgi
+    - python-psycopg2
+    - libsemanage-python
+  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+
+- name: ensure database is created
+  delegate_to: "{{ vault_db_host_machine }}"
+  become_user: postgres
+  become: true
+  postgresql_db: db={{ vault_db_name }}
+
+- name: ensure vault db user has access to database
+  delegate_to: "{{ vault_db_host_machine }}"
+  become_user: postgres
+  become: true
+  postgresql_user: db={{ vault_db_name }} user={{ vault_db_user }} password={{ vault_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: ensure selinux lets httpd talk to postgres
+  seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
+
+- name: generate vault config
+  template: src=settings.py.j2 dest=/etc/vault/settings.py owner=root group=root mode=0644
+  notify:
+    - reload httpd
+
+- name: generate vault apache config
+  template: src=vault.conf.j2 dest=/etc/httpd/conf.d/vault.conf owner=root group=root mode=0644
+  notify:
+    - reload httpd
+
+- name: generate alembic.ini
+  template: src=alembic.ini.j2 dest=/usr/share/vault/alembic.ini owner=root group=root mode=0644
+
+- name: initialize vault database
+  shell: PROD='true' vault init_db
+
+- name: initialize alembic
+  shell: PROD='true' vault init_alembic
+
+- name: upgrade vault database via alembic
+  shell: PROD='true' vault upgrade_db

roles/taskotron/vault/templates/alembic.ini.j2

@@ -0,0 +1,73 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions. When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+[alembic-packaged]
+# path to migration scripts on a packaged install
+script_location = /usr/share/vault/alembic
+
+sqlalchemy.url = 'postgresql+psycopg2://{{ vault_db_user }}:{{ vault_db_password }}@{{ vault_db_host }}:{{ vault_db_port }}/{{ vault_db_name }}'
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

roles/taskotron/vault/templates/settings.py.j2

@@ -0,0 +1,9 @@
+SECRET_KEY = '{{ vault_secret_key }}'
+SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ vault_db_user }}:{{ vault_db_password }}@{{ vault_db_host }}:{{ vault_db_port }}/{{ vault_db_name }}'
+
+FILE_LOGGING = False
+LOGFILE = '/var/log/vault/vault.log'
+SYSLOG_LOGGING = False
+STREAM_LOGGING = True
+
+MASTERKEY = '{{vault_masterkey}}'

roles/taskotron/vault/templates/vault.conf.j2

@@ -0,0 +1,34 @@
+WSGIDaemonProcess vault user=apache group=apache threads=5
+WSGIScriptAlias /{{ vault_endpoint }} /usr/share/vault/vault.wsgi
+WSGISocketPrefix run/wsgi
+
+# this isn't the best way to force SSL but it works for now
+#RewriteEngine On
+#RewriteCond %{HTTPS} !=on
+#RewriteRule ^/vault/admin/?(.*) https://%{SERVER_NAME}/$1 [R,L]
+
+<Directory /usr/share/vault>
+    WSGIProcessGroup vault
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIScriptReloading On
+    <IfModule mod_authz_core.c>
+        # Apache 2.4
+        <RequireAny>
+            Require method GET
+            Require ip 127.0.0.1 ::1{% for host in allowed_hosts %} {{ host }}{% endfor %}
+
+        </RequireAny>
+    </IfModule>
+    <IfModule !mod_auth_core.c>
+        Order allow,deny
+        Allow from all
+    </IfModule>
+
+</Directory>
+
+#Alias /vault/static /var/www/vault/vault/static
+
+#<Directory /var/www/vault/vault/static>
+#Order allow,deny
+#Allow from all
+#</Directory>