diff --git a/inventory/group_vars/resultsdb-dev b/inventory/group_vars/resultsdb-dev index 36c27423a1..116548c802 100644 --- a/inventory/group_vars/resultsdb-dev +++ b/inventory/group_vars/resultsdb-dev @@ -6,7 +6,7 @@ num_cpus: 4 # for systems that do not match the above - specify the same parameter in # the host_vars/$hostname file -tcp_ports: [ 80, 443, "{{ resultsdb_db_port }}" ] +tcp_ports: [ 80, 443, "{{ resultsdb_db_port }}", "{{ execdb_db_port }}" ] fas_client_groups: sysadmin-qa,sysadmin-main nrpe_procs_warn: 250 nrpe_procs_crit: 300 @@ -24,6 +24,12 @@ resultsdb_db_port: 5432 resultsdb_endpoint: 'resultsdb_api' resultsdb_fe_endpoint: 'resultsdb' resultsdb_db_name: resultsdb_dev + +execdb_db_host: db-qa01.qa.fedoraproject.org +execdb_db_port: 5432 +execdb_endpoint: 'execdb' +execdb_db_name: execdb_dev + allowed_hosts: - 10.5.124 diff --git a/inventory/group_vars/taskotron-dev b/inventory/group_vars/taskotron-dev index c3f8ecca19..fdf9168990 100644 --- a/inventory/group_vars/taskotron-dev +++ b/inventory/group_vars/taskotron-dev @@ -21,6 +21,7 @@ buildslave_dir: /home/buildslave/slave master_dir: /home/buildmaster/master master_user: buildmaster external_hostname: taskotron-dev.fedoraproject.org +execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbottest resultsdb_url: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb_api/api/v1.0 resultsdb_frontend_url: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb/ resultsdb_external_url: https://taskotron-dev.fedoraproject.org/resultsdb/ diff --git a/inventory/group_vars/taskotron-dev-clients b/inventory/group_vars/taskotron-dev-clients index b6650eea13..f40a460cc4 100644 --- a/inventory/group_vars/taskotron-dev-clients +++ b/inventory/group_vars/taskotron-dev-clients @@ -5,6 +5,7 @@ num_cpus: 2 slave_user: buildslave taskotron_fas_user: taskotron +execdb_server: http://resultsdb-dev01.qa.fedoraproject.org/execdb resultsdb_server: http://resultsdb-dev01.qa.fedoraproject.org/resultsdb_api/api/v1.0/ bodhi_server: http://10.5.124.181/fakefedorainfra/bodhi/ kojihub_url: http://koji.fedoraproject.org/kojihub diff --git a/inventory/host_vars/db-qa01.qa.fedoraproject.org b/inventory/host_vars/db-qa01.qa.fedoraproject.org index 504efc5a8a..c7353d3330 100644 --- a/inventory/host_vars/db-qa01.qa.fedoraproject.org +++ b/inventory/host_vars/db-qa01.qa.fedoraproject.org @@ -22,6 +22,7 @@ dbs_to_backup: - fakefedorainfra - fakefedorainfra_stg - dev_fakefedorainfra +- execdb_dev - resultsdb - resultsdb_stg - resultsdb_dev diff --git a/playbooks/groups/resultsdb-dev.yml b/playbooks/groups/resultsdb-dev.yml index 1ce76cec77..ad5f38a4b3 100644 --- a/playbooks/groups/resultsdb-dev.yml +++ b/playbooks/groups/resultsdb-dev.yml @@ -49,7 +49,7 @@ handlers: - include: "{{ handlers }}/restart_services.yml" -- name: configure resultsdb +- name: configure resultsdb and execdb hosts: resultsdb-dev user: root gather_facts: True @@ -62,6 +62,7 @@ roles: - { role: taskotron/resultsdb-backend, tags: ['resultsdb-be'] } - { role: taskotron/resultsdb-frontend, tags: ['resultsdb-fe'] } + - { role: taskotron/execdb, tags: ['execdb'] } handlers: - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 67ac2ebba4..ac56fb75bd 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -126,6 +126,12 @@ c['schedulers'].append(ForceScheduler( label='type of object to use', default='', size=256), +{% if deployment_type == 'dev' %} + StringParameter(name='uuid', + label='UUID of the build in progress', + default='', + size=256), +{% endif %} StringParameter(name='arch', label='arch of rpm to test', default='x86_64', @@ -158,9 +164,9 @@ factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }} mode='full', method='clobber')) # run the runner -factory.addStep(ShellCommand(command=["runtask", '-i', - Interpolate('%(prop:item)s'), '-t', - Interpolate('%(prop:item_type)s'), +factory.addStep(ShellCommand(command=["runtask", + '-i', Interpolate('%(prop:item)s'), + '-t', Interpolate('%(prop:item_type)s'), '-a', Interpolate('%(prop:arch)s'), '-j', Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'), {% if deployment_type == 'dev' %} @@ -356,6 +362,14 @@ mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', c['status'].append(mn) +{% if deployment_type == 'dev' %} +# ExecDB's push-notifications: +import buildbot.status.status_push +sp = buildbot.status.status_push.HttpStatusPush( + serverUrl="{{ execdb_statuspush }}", + debug=True) +c['status'].append(sp) +{% endif %} ####### PROJECT IDENTITY diff --git a/roles/taskotron/execdb/tasks/main.yml b/roles/taskotron/execdb/tasks/main.yml new file mode 100644 index 0000000000..31d8ddefc1 --- /dev/null +++ b/roles/taskotron/execdb/tasks/main.yml @@ -0,0 +1,57 @@ +- name: ensure packages required for execdb are installed + action: yum name={{ item }} state=latest + with_items: + - execdb + - mod_wsgi + - python-psycopg2 + - libsemanage-python + +- name: ensure database is created + delegate_to: "{{ execdb_db_host }}" + sudo_user: postgres + sudo: true + action: postgresql_db db={{ execdb_db_name }} + +- name: ensure dev execdb db user has access to dev database + when: deployment_type == 'dev' + delegate_to: "{{ execdb_db_host }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ execdb_db_name }} user={{ dev_execdb_db_user }} password={{ dev_execdb_db_password }} role_attr_flags=NOSUPERUSER + +- name: ensure stg execdb db user has access to stg database + when: deployment_type == 'stg' + delegate_to: "{{ execdb_db_host }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ execdb_db_name }} user={{ stg_execdb_db_user }} password={{ stg_execdb_db_password }} role_attr_flags=NOSUPERUSER + +- name: ensure prod execdb db user has access to prod database + when: deployment_type == 'prod' + delegate_to: "{{ execdb_db_host }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ execdb_db_name }} user={{ prod_execdb_db_user }} password={{ prod_execdb_db_password }} role_attr_flags=NOSUPERUSER + +- name: ensure local execdb db user has access to prod database + when: deployment_type == 'local' + delegate_to: "{{ execdb_db_host }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ execdb_db_name }} user={{ local_execdb_db_user }} password={{ local_execdb_db_password }} role_attr_flags=NOSUPERUSER + +- name: ensure selinux lets httpd talk to postgres + seboolean: name=httpd_can_network_connect_db persistent=yes state=yes + +- name: generate execdb config + template: src=settings.py.j2 dest=/etc/execdb/settings.py owner=root group=root mode=0644 + notify: + - restart httpd + +- name: generate execdb apache config + template: src=execdb.conf.j2 dest=/etc/httpd/conf.d/execdb.conf owner=root group=root mode=0644 + notify: + - restart httpd + +- name: initialize execdb database + shell: PROD='true' execdb init_db diff --git a/roles/taskotron/execdb/templates/execdb.conf.j2 b/roles/taskotron/execdb/templates/execdb.conf.j2 new file mode 100644 index 0000000000..cb52a054f5 --- /dev/null +++ b/roles/taskotron/execdb/templates/execdb.conf.j2 @@ -0,0 +1,34 @@ +WSGIDaemonProcess execdb user=apache group=apache threads=5 +WSGIScriptAlias /{{ execdb_endpoint }} /usr/share/execdb/execdb.wsgi +WSGISocketPrefix run/wsgi + +# this isn't the best way to force SSL but it works for now +#RewriteEngine On +#RewriteCond %{HTTPS} !=on +#RewriteRule ^/execdb/admin/?(.*) https://%{SERVER_NAME}/$1 [R,L] + + + WSGIProcessGroup execdb + WSGIApplicationGroup %{GLOBAL} + WSGIScriptReloading On + + # Apache 2.4 + + Require method GET + Require ip 127.0.0.1 ::1{% for host in allowed_hosts %} {{ host }}{% endfor %} + + + + + Order allow,deny + Allow from all + + + + +#Alias /execdb/static /var/www/execdb/execdb/static + +# +#Order allow,deny +#Allow from all +# diff --git a/roles/taskotron/execdb/templates/settings.py.j2 b/roles/taskotron/execdb/templates/settings.py.j2 new file mode 100644 index 0000000000..854bac94e7 --- /dev/null +++ b/roles/taskotron/execdb/templates/settings.py.j2 @@ -0,0 +1,20 @@ +{%- if deployment_type == 'prod' %} +SECRET_KEY = '{{ prod_execdb_secret_key }}' +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ prod_execdb_db_user }}:{{ prod_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}' +{% endif %} +{%- if deployment_type == 'stg' %} +SECRET_KEY = '{{ stg_execdb_secret_key }}' +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ stg_execdb_db_user }}:{{ stg_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}' +{% endif %} +{%- if deployment_type == 'dev' %} +SECRET_KEY = '{{ dev_execdb_secret_key }}' +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ dev_execdb_db_user }}:{{ dev_execdb_db_password }}@{{ execdb_db_host }}:{{ execdb_db_port }}/{{ execdb_db_name }}' +{% endif %} +{%- if deployment_type == 'local' %} +SECRET_KEY = '{{ local_execdb_secret_key }}' +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{{ local_execdb_db_user }}:{{ local_execdb_db_password }}@127.0.0.1:{{ execdb_db_port }}/{{ execdb_db_name }}' +{% endif %} +FILE_LOGGING = False +LOGFILR = '/var/log/execdb/execdb.log' +SYSLOG_LOGGING = False +STREAM_LOGGING = True diff --git a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 index 379aba77eb..a4155321e6 100644 --- a/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 +++ b/roles/taskotron/taskotron-client/templates/taskotron.yaml.j2 @@ -55,6 +55,9 @@ bodhi_server: {{ bodhi_server }} ## URL of ResultsDB server API interface, which can store all test results resultsdb_server: {{ resultsdb_server }} +## URL of ExecDB server API interface, which tracks task execution status +execdb_server: {{ execdb_server }} + ## URL of taskotron buildmaster taskotron_master: {{ taskotron_master }}