Deploy ExecDB to the ResultsDB-dev-machine

inventory/group_vars/resultsdb-dev

@@ -6,7 +6,7 @@ num_cpus: 4
 # for systems that do not match the above - specify the same parameter in
 # the host_vars/$hostname file
 
-tcp_ports: [ 80, 443, "{{ resultsdb_db_port }}" ]
+tcp_ports: [ 80, 443, "{{ resultsdb_db_port }}", "{{ execdb_db_port }}" ]
 fas_client_groups: sysadmin-qa,sysadmin-main
 nrpe_procs_warn: 250
 nrpe_procs_crit: 300
@@ -24,6 +24,12 @@ resultsdb_db_port: 5432
 resultsdb_endpoint: 'resultsdb_api'
 resultsdb_fe_endpoint: 'resultsdb'
 resultsdb_db_name: resultsdb_dev
+
+execdb_db_host: db-qa01.qa.fedoraproject.org
+execdb_db_port: 5432
+execdb_endpoint: 'execdb'
+execdb_db_name: execdb_dev
+
 allowed_hosts:
     - 10.5.124
 

inventory/group_vars/taskotron-dev

@@ -21,6 +21,7 @@ buildslave_dir: /home/buildslave/slave
 master_dir: /home/buildmaster/master
 master_user: buildmaster
 external_hostname: taskotron-dev.fedoraproject.org
+execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbottest
 resultsdb_url: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb_api/api/v1.0
 resultsdb_frontend_url: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb/
 resultsdb_external_url: https://taskotron-dev.fedoraproject.org/resultsdb/

inventory/group_vars/taskotron-dev-clients

@@ -5,6 +5,7 @@ num_cpus: 2
 
 slave_user: buildslave
 taskotron_fas_user: taskotron
+execdb_server: http://resultsdb-dev01.qa.fedoraproject.org/execdb
 resultsdb_server: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb_api/api/v1.0/
 bodhi_server: http://10.5.124.181/fakefedorainfra/bodhi/
 kojihub_url: http://koji.fedoraproject.org/kojihub

inventory/host_vars/db-qa01.qa.fedoraproject.org

@@ -22,6 +22,7 @@ dbs_to_backup:
 - fakefedorainfra
 - fakefedorainfra_stg
 - dev_fakefedorainfra
+- execdb_dev
 - resultsdb
 - resultsdb_stg
 - resultsdb_dev

playbooks/groups/resultsdb-dev.yml

@@ -49,7 +49,7 @@
   handlers:
    - include: "{{ handlers }}/restart_services.yml"
 
-- name: configure resultsdb
+- name: configure resultsdb and execdb
   hosts: resultsdb-dev
   user: root
   gather_facts: True
@@ -62,6 +62,7 @@
   roles:
    - { role: taskotron/resultsdb-backend, tags: ['resultsdb-be'] }
    - { role: taskotron/resultsdb-frontend, tags: ['resultsdb-fe'] }
+   - { role: taskotron/execdb, tags: ['execdb'] }
 
   handlers:
    - include: "{{ handlers }}/restart_services.yml"

roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2

@@ -126,6 +126,12 @@ c['schedulers'].append(ForceScheduler(
                                             label='type of object to use',
                                             default='',
                                             size=256),
+{% if deployment_type == 'dev' %}
+                            StringParameter(name='uuid',
+                                            label='UUID of the build in progress',
+                                            default='',
+                                            size=256),
+{% endif %}
                             StringParameter(name='arch',
                                             label='arch of rpm to test',
                                             default='x86_64',
@@ -158,9 +164,9 @@ factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}
                     mode='full', method='clobber'))
 
 # run the runner
-factory.addStep(ShellCommand(command=["runtask", '-i',
+factory.addStep(ShellCommand(command=["runtask",
-                                        Interpolate('%(prop:item)s'), '-t',
+                                        '-i', Interpolate('%(prop:item)s'),
-                                        Interpolate('%(prop:item_type)s'),
+                                        '-t', Interpolate('%(prop:item_type)s'),
                                         '-a', Interpolate('%(prop:arch)s'),
                                         '-j', Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'),
 {% if deployment_type == 'dev' %}
@@ -356,6 +362,14 @@ mn = MailNotifier(fromaddr='taskotron@fedoraproject.org',
 
 c['status'].append(mn)
 
+{% if deployment_type == 'dev' %}
+# ExecDB's push-notifications:
+import buildbot.status.status_push
+sp = buildbot.status.status_push.HttpStatusPush(
+        serverUrl="{{ execdb_statuspush }}",
+        debug=True)
+c['status'].append(sp)
+{% endif %}
 
 ####### PROJECT IDENTITY
 

roles/taskotron/execdb/tasks/main.yml

@@ -0,0 +1,57 @@
+- name: ensure packages required for execdb are installed
+  action: yum name={{ item }} state=latest
+  with_items:
+    - execdb
+    - mod_wsgi
+    - python-psycopg2
+    - libsemanage-python
+
+- name: ensure database is created
+  delegate_to: "{{ execdb_db_host }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_db db={{ execdb_db_name }}
+
+- name: ensure dev execdb db user has access to dev database
+  when: deployment_type == 'dev'
+  delegate_to: "{{ execdb_db_host }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_user db={{ execdb_db_name }} user={{ dev_execdb_db_user }} password={{ dev_execdb_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: ensure stg execdb db user has access to stg database
+  when: deployment_type == 'stg'
+  delegate_to: "{{ execdb_db_host }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_user db={{ execdb_db_name }} user={{ stg_execdb_db_user }} password={{ stg_execdb_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: ensure prod execdb db user has access to prod database
+  when: deployment_type == 'prod'
+  delegate_to: "{{ execdb_db_host }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_user db={{ execdb_db_name }} user={{ prod_execdb_db_user }} password={{ prod_execdb_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: ensure local execdb db user has access to prod database
+  when: deployment_type == 'local'
+  delegate_to: "{{ execdb_db_host }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_user db={{ execdb_db_name }} user={{ local_execdb_db_user }} password={{ local_execdb_db_password }} role_attr_flags=NOSUPERUSER
+
+- name: ensure selinux lets httpd talk to postgres
+  seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
+
+- name: generate execdb config
+  template: src=settings.py.j2 dest=/etc/execdb/settings.py owner=root group=root mode=0644
+  notify:
+    - restart httpd
+
+- name: generate execdb apache config
+  template: src=execdb.conf.j2 dest=/etc/httpd/conf.d/execdb.conf owner=root group=root mode=0644
+  notify:
+    - restart httpd
+
+- name: initialize execdb database
+  shell: PROD='true' execdb init_db

roles/taskotron/execdb/templates/execdb.conf.j2

@@ -0,0 +1,34 @@
+WSGIDaemonProcess execdb user=apache group=apache threads=5
+WSGIScriptAlias /{{ execdb_endpoint }} /usr/share/execdb/execdb.wsgi
+WSGISocketPrefix run/wsgi
+
+# this isn't the best way to force SSL but it works for now
+#RewriteEngine On
+#RewriteCond %{HTTPS} !=on
+#RewriteRule ^/execdb/admin/?(.*) https://%{SERVER_NAME}/$1 [R,L]
+
+<Directory /usr/share/execdb>
+    WSGIProcessGroup execdb
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIScriptReloading On
+    <IfModule mod_authz_core.c>
+        # Apache 2.4
+        <RequireAny>
+            Require method GET
+            Require ip 127.0.0.1 ::1{% for host in allowed_hosts %} {{ host }}{% endfor %}
+
+        </RequireAny>
+    </IfModule>
+    <IfModule !mod_auth_core.c>
+        Order allow,deny
+        Allow from all
+    </IfModule>
+
+</Directory>
+
+#Alias /execdb/static /var/www/execdb/execdb/static
+
+#<Directory /var/www/execdb/execdb/static>
+#Order allow,deny
+#Allow from all
+#</Directory>

roles/taskotron/execdb/templates/settings.py.j2

@@ -0,0 +1,20 @@
+{%- if deployment_type == 'prod' %}
+SECRET_KEY = '{{ prod_execdb_secret_key }}'
+SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ prod_execdb_db_user }}:{{ prod_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}'
+{% endif %}
+{%- if deployment_type == 'stg' %}
+SECRET_KEY = '{{ stg_execdb_secret_key }}'
+SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ stg_execdb_db_user }}:{{ stg_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}'
+{% endif %}
+{%- if deployment_type == 'dev' %}
+SECRET_KEY = '{{ dev_execdb_secret_key }}'
+SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ dev_execdb_db_user }}:{{ dev_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}'
+{% endif %}
+{%- if deployment_type == 'local' %}
+SECRET_KEY = '{{ local_execdb_secret_key }}'
+SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ local_execdb_db_user }}:{{ local_execdb_db_password }}@127.0.0.1:{{ execdb_db_port }}/{{ execdb_db_name }}'
+{% endif %}
+FILE_LOGGING = False
+LOGFILR = '/var/log/execdb/execdb.log'
+SYSLOG_LOGGING = False
+STREAM_LOGGING = True

roles/taskotron/taskotron-client/templates/taskotron.yaml.j2

@@ -55,6 +55,9 @@ bodhi_server: {{ bodhi_server }}
 ## URL of ResultsDB server API interface, which can store all test results
 resultsdb_server: {{ resultsdb_server }}
 
+## URL of ExecDB server API interface, which tracks task execution status
+execdb_server: {{ execdb_server }}
+
 ## URL of taskotron buildmaster
 taskotron_master: {{ taskotron_master }}