set up for openQA deployment

This adds openQA server, worker and dispatcher roles, and
applies them to the appropriate hosts. A few secret vars are
required. See trac #4958 for discussion.

inventory/group_vars/openqa

@@ -0,0 +1,18 @@
+external_hostname: openqa.fedoraproject.org
+
+openqa_hostname: localhost
+openqa_email: adamwill@fedoraproject.org
+openqa_nickname: adamwill
+openqa_fullname: Adam Williamson
+openqa_userid: http://adamwill.id.fedoraproject.org/
+
+openqa_dbname: openqa
+openqa_dbhost: db-qa01.qa.fedoraproject.org
+openqa_dbuser: openqa
+
+wikitcms_user: coconut
+
+openqa_workers: 3
+
+# NFS
+tcp_ports: 2049

inventory/group_vars/openqa-stg

@@ -0,0 +1,19 @@
+external_hostname: openqa-stg.fedoraproject.org
+
+openqa_hostname: localhost
+openqa_email: adamwill@fedoraproject.org
+openqa_nickname: adamwill
+openqa_fullname: Adam Williamson
+openqa_userid: http://adamwill.id.fedoraproject.org/
+
+openqa_dbname: openqa-stg
+openqa_dbhost: db-qa01.qa.fedoraproject.org
+openqa_dbuser: openqa
+
+openqa_workers: 3
+
+# For now let's not run any scheduled jobs on stg
+openqa_triggers: []
+
+# NFS
+tcp_ports: 2049

inventory/group_vars/openqa-stg-workers

@@ -0,0 +1,2 @@
+openqa_workers: 4
+openqa_hostname: {{ groups['openqa-stg'][0] }}

inventory/group_vars/openqa-workers

@@ -0,0 +1,2 @@
+openqa_workers: 4
+openqa_hostname: {{ groups['openqa'][0] }}

playbooks/groups/openqa-workers.yml

@@ -1,10 +1,4 @@
-# create a new openqa worker server system
-# NOTE: should be used with --limit most of the time
-# NOTE: most of these vars_path come from group_vars/backup_server or from hostvars
-# This has an extra role that configures the virthost to be used with beaker for
-# virtual machine clients
-
-- name: basic configuration
+- name: configure openQA workers
   hosts: openqa-workers:openqa-stg-workers
   user: root
   gather_facts: True
@@ -15,16 +9,15 @@
    - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
   roles:
-  - base
-  - rkhunter
-  - { role: denyhosts, when: ansible_distribution_major_version|int != 7 }
-  - nagios_client
-  - hosts
-  - fas_client
-  - collectd/base
-  - { role: iscsi_client, when: datacenter == "phx2" }
-  - sudo
-  - { role: openvpn/client, when: datacenter != "phx2" }
+   - { role: base, tags: ['base'] }
+   - { role: rkhunter, tags: ['rkhunter'] }
+   - { role: nagios_client, tags: ['nagios_client'] }
+   - { role: hosts, tags: ['hosts']}
+   - { role: fas_client, tags: ['fas_client'] }
+   - { role: collectd/base, tags: ['collectd_base'] }
+   - { role: sudo, tags: ['sudo'] }
+   - { role: openqa_worker, tags: ['openqa_worker'] }
+   - apache
 
   tasks:
   - include: "{{ tasks }}/yumrepos.yml"
@@ -32,20 +25,4 @@
   - include: "{{ tasks }}/motd.yml"
 
   handlers:
-  - include: "{{ handlers }}/restart_services.yml"
-
-#- name: configure openqa workers
-#  hosts: openqa-workers:openqa-stg-workers
-#  user: root
-#  gather_facts: True
-#
-#  vars_files:
-#   - /srv/web/infra/ansible/vars/global.yml
-#   - "/srv/private/ansible/vars.yml"
-#   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
-#
-#  roles:
-#   - { role: openqa/something, tags: ['something'] }
-#
-#  handlers:
-#   - include: "{{ handlers }}/restart_services.yml"
+   - include: "{{ handlers }}/restart_services.yml"

playbooks/groups/openqa.yml

@@ -1,11 +1,4 @@
----
-# create a new taskotron staging server
-# NOTE: make sure there is room/space for this server on the vmhost
-# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars
-
-- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=openqa:openqa-stg"
-
-- name: make the box be real
+- name: configure openQA
   hosts: openqa:openqa-stg
   user: root
   gather_facts: True
@@ -22,31 +15,16 @@
    - { role: hosts, tags: ['hosts']}
    - { role: fas_client, tags: ['fas_client'] }
    - { role: collectd/base, tags: ['collectd_base'] }
-#   - { role: yum-cron, tags: ['yumcron'] }
    - { role: sudo, tags: ['sudo'] }
+   - { role: openqa/server, tags: ['openqa_server'] }
+   - { role: openqa/worker, tags: ['openqa_worker'] }
+   - { role: openqa/dispatcher, tags: ['openqa_dispatcher'] }
    - apache
 
   tasks:
-  # this is how you include other task lists
   - include: "{{ tasks }}/yumrepos.yml"
   - include: "{{ tasks }}/2fa_client.yml"
   - include: "{{ tasks }}/motd.yml"
 
   handlers:
    - include: "{{ handlers }}/restart_services.yml"
-
-#- name: configure openqa
-#  hosts: openqa:openqa-stg
-#  user: root
-#  gather_facts: True
-#
-#  vars_files:
-#   - /srv/web/infra/ansible/vars/global.yml
-#   - "/srv/private/ansible/vars.yml"
-#   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
-#
-#  roles:
-#   - { role: openqa/something, tags: ['something'] }
-#
-#  handlers:
-#   - include: "{{ handlers }}/restart_services.yml"

roles/openqa/dispatcher/tasks/main.yml

@@ -0,0 +1,99 @@
+# Required vars with defaults
+# - openqa_hostname
+##      string - hostname of openQA server to run jobs for
+##      default - localhost
+# - openqa_triggers
+##      list - the timers to enable - i.e. which scheduled runs
+##             will actually be triggered by this dispatcher
+##      default - ['current', 'branched', 'rawhide', 'stable']
+#
+# Optional vars
+# - wikitcms_user
+##      string - FAS username for reporting results to wiki
+# - wikitcms_password
+##      string - password for relval_user
+#
+# When both of the above are set, a wikitcms 'credentials' file will
+# be created and result submission to the wiki will be enabled.
+#
+# NOTE: At present, as the scheduler code downloads the ISOs, it only
+# really makes sense for openqa_server boxes to be their own
+# openqa_dispatchers. However, this should be fixed in the scheduler
+# code soon, so the roles are kept separate to give future flexibility.
+
+- name: Install required packages
+  dnf: name={{ item }} state=present
+  with_items:
+  - fedfind
+  - python-wikitcms
+  - python-requests
+  - python-setuptools
+  - python-six
+  tags:
+  - packages
+
+- name: Check out openQA-python-client
+  git:
+    repo: https://github.com/os-autoinst/openQA-python-client.git
+    dest: /root/openQA-python-client
+  register: gitclient
+
+- name: Check if python-client has ever been installed
+  command: "python -c 'import openqa_client'"
+  register: instclient
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+
+- name: Install openQA-python-client
+  command: "python setup.py install"
+  args:
+    chdir: /root/openQA-python-client
+  when: "gitclient|changed or instclient.rc > 0"
+
+- name: Check out openqa_fedora_tools
+  git:
+    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
+    dest: /root/openqa_fedora_tools
+  register: gittools
+
+- name: Check if openqa_fedora_tools has ever been installed
+  stat: path=/usr/bin/fedora-openqa-schedule
+  register: insttools
+  changed_when: "1 != 1"
+  failed_when: "1 != 1"
+
+- name: Install openqa_fedora_tools
+  command: "python setup.py install"
+  args:
+    chdir: /root/openqa_fedora_tools
+  when: "gittools|changed or not insttools.stat.exists"
+
+- name: Reload systemd config to pick up installed units
+  command: "systemctl daemon-reload"
+  when: "gittools|changed or not insttools.stat.exists"
+
+- name: openQA client config
+  template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600
+  tags:
+  - config
+
+- name: Create fedora-openqa-schedule config directory
+  file: path=/etc/fedora-qa state=directory owner=root group=root mode=0700
+
+- name: Write schedule.conf
+  template: src=schedule.conf.j2 dest=/etc/fedora-qa/schedule.conf owner=root group=root mode=0600
+  tags:
+  - config
+
+- name: Create /root/.fedora (credentials files location)
+  file: path=/root/.fedora state=directory owner=root group=root mode=0700
+
+- name: Write wikitcms credentials file
+  template: src=credentials.j2 dest=/root/.fedora/credentials owner=root group=root mode=0600
+  when: "wikitcms_user is defined and wikitcms_password is defined"
+  tags:
+  - config
+
+- name: Enable and start timers
+  service: name=openqa-trigger-{{ item }}.timer enabled=yes state=started
+  with_items: "{{ openqa_triggers }}"

roles/openqa/dispatcher/templates/client.conf.j2

@@ -0,0 +1,3 @@
+[{{ openqa_hostname|default('localhost') }}]
+key = {{ openqa_key }}
+secret = {{ openqa_secret }}

roles/openqa/dispatcher/templates/credentials.j2

@@ -0,0 +1 @@
+{{ wikitcms_user }} {{ wikitcms_password }}

roles/openqa/dispatcher/templates/schedule.conf.j2

@@ -0,0 +1,6 @@
+[report]
+{% if wikitcms_user is defined and wikitcms_password is defined %}
+submit: true
+{% else %}
+submit: false
+{% endif %}

roles/openqa/dispatcher/vars/main.yml

@@ -0,0 +1,6 @@
+openqa_hostname: localhost
+openqa_triggers:
+  - current
+  - branched
+  - rawhide
+  - stable

roles/openqa/server/files/exports

@@ -0,0 +1 @@
+/var/lib/openqa/share *(ro,insecure,all_squash)

roles/openqa/server/files/openqa.ini

@@ -0,0 +1,12 @@
+[global]
+branding = plain
+
+[auth]
+method=OpenID
+
+[logging]
+level=info
+
+[openid]
+provider = https://id.fedoraproject.org/
+httpsonly = 1

roles/openqa/server/tasks/main.yml

@@ -0,0 +1,167 @@
+# Required vars
+# - openqa_email
+##       string - Email address of admin user
+# - openqa_nickname
+##       string - Short name of admin user (shown in the web UI for e.g.)
+# - openqa_fullname
+##       string - Full name of admin user
+# - openqa_key
+# - openqa_secret
+##       string - MUST be 16-character hexadecimals, and are secrets
+# openqa_userid
+##       string - User ID of admin user: for Fedora should be a Fedora openID URL,
+##                http://fasname.id.fedoraproject.org
+
+# Required vars with defaults
+# - external_hostname
+##       string - The public hostname for the server (will be used as ServerName)
+##       default - ansible_nodename
+
+# Optional vars
+# - openqa_dbname
+##       string - The name of the database to use
+# - openqa_dbhost
+##       string - The hostname of the database server
+# - openqa_dbuser
+##       string - The database username
+# - openqa_dbpassword
+##       string - The database password
+#
+# If openqa_dbhost is set, the others must be too, and the server will be
+# configured to use a pgsql database accordingly. If openqa_dbhost is not
+# set, the server will use a local SQLite database and the other values
+# are ignored.
+
+- name: Ensure DNF COPR plugin is available
+  dnf: pkg="dnf-command(copr)" state=present
+  tags:
+  - packages
+
+- name: Install openQA repo if needed
+  command: "dnf -y copr enable adamwill/openQA"
+  args:
+    creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo
+  tags:
+  - config
+
+- name: Install required packages
+  dnf: name={{ item }} state=present enablerepo=adamwill-openQA
+  with_items:
+  - libselinux-python
+  - openqa
+  - git
+  - libselinux-utils
+  - libsemanage-python
+  - nfs-utils
+  - perl(Class::DBI::Pg)
+  - perl(DateTime::Format::Pg)
+  - libguestfs-tools-c
+  tags:
+  - packages
+
+- name: Check out the tests
+  git:
+    repo: https://bitbucket.org/rajcze/openqa_fedora
+    dest: /var/lib/openqa/share/tests/fedora
+
+- name: Have tests owned by geekotest
+  file: path=/var/lib/openqa/share/tests/fedora owner=geekotest recurse=yes
+
+- name: Check out openqa_fedora_tools
+  git:
+    repo: https://bitbucket.org/rajcze/openqa_fedora_tools
+    dest: /root/openqa_fedora_tools
+
+- name: Create asset directories
+  file: path={{ item }} state=directory owner=geekotest group=root mode=0755
+  with_items:
+  - /var/lib/openqa/share/factory/iso
+  - /var/lib/openqa/share/factory/hdd
+  - /var/lib/openqa/share/factory/repo
+
+# NOTE: this is very hacky, but we can't do much better with the current
+# disk creation script, I will try and make it better. We'll have to bump
+# this hardcoded release number every so often.
+- name: Create hard disk images (this may take a long time!)
+  command: "/root/openqa_fedora_tools/tools/createhdds.sh 23"
+  args:
+    creates: /var/lib/openqa/share/factory/hdd/disk_full.img
+    chdir: /var/lib/openqa/share/factory/hdd/
+
+- name: Create exports file
+  copy: src=exports dest=/etc/exports.d/openqa.exports owner=root group=root mode=0644
+  tags:
+  - config
+
+- name: Enable and start NFS server
+  service: name=nfs-server enabled=yes state=started
+
+- name: Set up Apache config
+  template: src=openqa.conf.httpd.j2 dest=/etc/httpd/conf.d/openqa.conf owner=root group=root mode=0644
+  tags:
+  - config
+
+- name: OpenQA config
+  copy: src=openqa.ini dest=/etc/openqa/openqa.ini owner=geekotest group=root mode=0640
+  tags:
+  - config
+
+- name: Create database
+  delegate_to: "{{ openqa_dbhost }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_db db={{ openqa_dbname }}
+  when: "openqa_dbhost is defined"
+
+- name: Ensure db user has access to database
+  delegate_to: "{{ openqa_dbhost }}"
+  sudo_user: postgres
+  sudo: true
+  action: postgresql_user db={{ openqa_dbname }} user={{ openqa_dbuser }} password={{ openqa_dbpassword }} role_attr_flags=NOSUPERUSER
+  when: "openqa_dbhost is defined"
+
+- name: Database config
+  template: src=database.ini.pgsql.j2 dest=/etc/openqa/database.ini owner=geekotest group=root mode=0640
+  when: "openqa_dbhost is defined"
+  tags:
+  - config
+
+- name: Initialize database
+  command: "/usr/share/openqa/script/initdb --user geekotest --init_database"
+  register: initdb
+  changed_when: "initdb.rc == 0"
+  failed_when: "(initdb.rc > 0) and (initdb.stderr is not defined or initdb.stderr.find('already exists') == -1)"
+
+- name: Enable and start services
+  service: name={{ item }} enabled=yes state=started
+  register: services
+  with_items:
+  - openqa-scheduler
+  - openqa-webui
+  - openqa-websockets
+  - openqa-gru
+
+# This is using a big hammer until #1277312 is resolved
+- name: Allow Apache to connect to openQA
+  seboolean: name=httpd_can_network_connect state=yes persistent=yes
+
+- name: Wait for openQA to be fully started
+  pause: seconds=5
+  when: services|changed
+
+- name: openQA client config
+  template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600
+  tags:
+  - config
+
+- name: Create admin user
+  command: "/var/lib/openqa/script/create_admin --email {{ openqa_email }} --nickname {{ openqa_nickname }} --fullname '{{ openqa_fullname }}' --key {{ openqa_key }} --secret {{ openqa_secret }} {{ openqa_userid }}"
+  register: admin
+  changed_when: "admin.rc == 0"
+  failed_when: "(admin.rc > 0) and (admin.stderr is not defined or admin.stderr.find('already exists') == -1)"
+
+- name: Load tests
+  shell: "/var/lib/openqa/share/tests/fedora/templates | grep 'added => [1-9]'"
+  register: templates
+  changed_when: "templates.rc == 0"
+  failed_when: "1 != 1"

roles/openqa/server/templates/client.conf.j2

@@ -0,0 +1,3 @@
+[{{ openqa_hostname|default('localhost') }}]
+key = {{ openqa_key }}
+secret = {{ openqa_secret }}

roles/openqa/server/templates/database.ini.pgsql.j2

@@ -0,0 +1,9 @@
+[test]
+dsn = dbi:SQLite:dbname=:memory:
+on_connect_call = use_foreign_keys
+on_connect_do = PRAGMA synchronous = OFF
+
+[production]
+dsn = dbi:Pg:dbname={{ openqa_dbname }};host={{ openqa_dbhost }}
+user = {{ openqa_dbuser }}
+password = {{ openqa_dbpassword }}

roles/openqa/server/templates/openqa.conf.httpd.j2

@@ -0,0 +1,4 @@
+<VirtualHost *:80>
+    ServerName {{ external_hostname|default(ansible_nodename) }}
+    Include conf.d/openqa-common.inc
+</VirtualHost>

roles/openqa/worker/tasks/main.yml

@@ -0,0 +1,45 @@
+# Required vars
+# - openqa_workers
+##      integer - number of worker instances to create/run
+
+# Required vars with defaults
+# - openqa_hostname
+##      string - hostname of openQA server to run jobs for
+##      default - localhost
+
+- name: Ensure DNF COPR plugin is available
+  dnf: pkg="dnf-command(copr)" state=present
+  tags:
+  - packages
+
+- name: Install openQA repo if needed
+  command: "dnf -y copr enable adamwill/openQA"
+  args:
+    creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo
+  tags:
+  - config
+
+- name: Install packages
+  dnf: name={{ item }} state=present enablerepo=adamwill-openQA
+  with_items:
+  - openqa-worker
+  - libselinux-python
+  tags:
+  - packages
+
+- include: nfs-client.yml
+  when: openqa_hostname is defined and openqa_hostname != "localhost"
+
+- name: openQA client config
+  template: src=client.conf.j2 dest=/etc/openqa/client.conf owner=_openqa-worker group=root mode=0600
+  tags:
+  - config
+
+- name: openQA worker config
+  template: src=workers.ini.j2 dest=/etc/openqa/workers.ini owner=_openqa-worker group=root mode=0644
+  tags:
+  - config
+
+- name: Worker services
+  service: name=openqa-worker@{{ item }} enabled=yes state=started
+  with_sequence: "count={{ openqa_workers }}"

roles/openqa/worker/tasks/nfs-client.yml

@@ -0,0 +1,24 @@
+# Required vars
+# - openqa_hostname
+##      string - hostname of the openQA server (we assume it is hosting the NFS mount)
+
+- name: Install NFS client
+  dnf: name=nfs-utils state=present
+  tags:
+  - packages
+
+# We don't check ownership as, after mounting, it's owned by whatever the
+# UID of geekotest is on the server
+- name: Ensure mount target exists
+  file: path=/var/lib/openqa/share state=directory mode=0755
+
+- name: Create mount unit
+  template: src=var-lib-openqa-share.mount.j2 dest=/etc/systemd/system/var-lib-openqa-share.mount owner=root group=root mode=0644
+  tags:
+  - config
+
+- name: Enable and start mount
+  service: name={{ item }} enabled=yes state=started
+  with_items:
+  - var-lib-openqa-share.mount
+  - remote-fs.target

roles/openqa/worker/templates/client.conf.j2

@@ -0,0 +1,3 @@
+[{{ openqa_hostname|default('localhost') }}]
+key = {{ openqa_key }}
+secret = {{ openqa_secret }}

roles/openqa/worker/templates/var-lib-openqa-share.mount.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=openQA shared assets
+ConditionPathExists=/var/lib/openqa/share
+
+[Mount]
+What={{ openqa_hostname }}:/var/lib/openqa/share
+Where=/var/lib/openqa/share
+Type=nfs
+
+[Install]
+WantedBy = remote-fs.target

roles/openqa/worker/templates/workers.ini.j2

@@ -0,0 +1,2 @@
+[global]
+HOST = http://{{ openqa_hostname|default('localhost') }}

roles/openqa/worker/vars/main.yml

@@ -0,0 +1 @@
+openqa_hostname: localhost