diff --git a/inventory/group_vars/openqa b/inventory/group_vars/openqa index e69de29bb2..81c5faae8d 100644 --- a/inventory/group_vars/openqa +++ b/inventory/group_vars/openqa @@ -0,0 +1,18 @@ +external_hostname: openqa.fedoraproject.org + +openqa_hostname: localhost +openqa_email: adamwill@fedoraproject.org +openqa_nickname: adamwill +openqa_fullname: Adam Williamson +openqa_userid: http://adamwill.id.fedoraproject.org/ + +openqa_dbname: openqa +openqa_dbhost: db-qa01.qa.fedoraproject.org +openqa_dbuser: openqa + +wikitcms_user: coconut + +openqa_workers: 3 + +# NFS +tcp_ports: 2049 diff --git a/inventory/group_vars/openqa-stg b/inventory/group_vars/openqa-stg index e69de29bb2..7620270180 100644 --- a/inventory/group_vars/openqa-stg +++ b/inventory/group_vars/openqa-stg @@ -0,0 +1,19 @@ +external_hostname: openqa-stg.fedoraproject.org + +openqa_hostname: localhost +openqa_email: adamwill@fedoraproject.org +openqa_nickname: adamwill +openqa_fullname: Adam Williamson +openqa_userid: http://adamwill.id.fedoraproject.org/ + +openqa_dbname: openqa-stg +openqa_dbhost: db-qa01.qa.fedoraproject.org +openqa_dbuser: openqa + +openqa_workers: 3 + +# For now let's not run any scheduled jobs on stg +openqa_triggers: [] + +# NFS +tcp_ports: 2049 diff --git a/inventory/group_vars/openqa-stg-workers b/inventory/group_vars/openqa-stg-workers new file mode 100644 index 0000000000..0f56cd18b7 --- /dev/null +++ b/inventory/group_vars/openqa-stg-workers @@ -0,0 +1,2 @@ +openqa_workers: 4 +openqa_hostname: {{ groups['openqa-stg'][0] }} diff --git a/inventory/group_vars/openqa-workers b/inventory/group_vars/openqa-workers new file mode 100644 index 0000000000..9f8c683642 --- /dev/null +++ b/inventory/group_vars/openqa-workers @@ -0,0 +1,2 @@ +openqa_workers: 4 +openqa_hostname: {{ groups['openqa'][0] }} diff --git a/playbooks/groups/openqa-workers.yml b/playbooks/groups/openqa-workers.yml index 6425876202..5be3f43346 100644 --- a/playbooks/groups/openqa-workers.yml +++ b/playbooks/groups/openqa-workers.yml @@ -1,30 +1,23 @@ -# create a new openqa worker server system -# NOTE: should be used with --limit most of the time -# NOTE: most of these vars_path come from group_vars/backup_server or from hostvars -# This has an extra role that configures the virthost to be used with beaker for -# virtual machine clients - -- name: basic configuration +- name: configure openQA workers hosts: openqa-workers:openqa-stg-workers user: root gather_facts: True - vars_files: + vars_files: - /srv/web/infra/ansible/vars/global.yml - "/srv/private/ansible/vars.yml" - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: - - base - - rkhunter - - { role: denyhosts, when: ansible_distribution_major_version|int != 7 } - - nagios_client - - hosts - - fas_client - - collectd/base - - { role: iscsi_client, when: datacenter == "phx2" } - - sudo - - { role: openvpn/client, when: datacenter != "phx2" } + - { role: base, tags: ['base'] } + - { role: rkhunter, tags: ['rkhunter'] } + - { role: nagios_client, tags: ['nagios_client'] } + - { role: hosts, tags: ['hosts']} + - { role: fas_client, tags: ['fas_client'] } + - { role: collectd/base, tags: ['collectd_base'] } + - { role: sudo, tags: ['sudo'] } + - { role: openqa_worker, tags: ['openqa_worker'] } + - apache tasks: - include: "{{ tasks }}/yumrepos.yml" @@ -32,20 +25,4 @@ - include: "{{ tasks }}/motd.yml" handlers: - - include: "{{ handlers }}/restart_services.yml" - -#- name: configure openqa workers -# hosts: openqa-workers:openqa-stg-workers -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - { role: openqa/something, tags: ['something'] } -# -# handlers: -# - include: "{{ handlers }}/restart_services.yml" + - include: "{{ handlers }}/restart_services.yml" diff --git a/playbooks/groups/openqa.yml b/playbooks/groups/openqa.yml index 158be787b1..3bc34e5ae6 100644 --- a/playbooks/groups/openqa.yml +++ b/playbooks/groups/openqa.yml @@ -1,11 +1,4 @@ ---- -# create a new taskotron staging server -# NOTE: make sure there is room/space for this server on the vmhost -# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars - -- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=openqa:openqa-stg" - -- name: make the box be real +- name: configure openQA hosts: openqa:openqa-stg user: root gather_facts: True @@ -22,31 +15,16 @@ - { role: hosts, tags: ['hosts']} - { role: fas_client, tags: ['fas_client'] } - { role: collectd/base, tags: ['collectd_base'] } -# - { role: yum-cron, tags: ['yumcron'] } - { role: sudo, tags: ['sudo'] } + - { role: openqa/server, tags: ['openqa_server'] } + - { role: openqa/worker, tags: ['openqa_worker'] } + - { role: openqa/dispatcher, tags: ['openqa_dispatcher'] } - apache tasks: - # this is how you include other task lists - include: "{{ tasks }}/yumrepos.yml" - include: "{{ tasks }}/2fa_client.yml" - include: "{{ tasks }}/motd.yml" handlers: - include: "{{ handlers }}/restart_services.yml" - -#- name: configure openqa -# hosts: openqa:openqa-stg -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - { role: openqa/something, tags: ['something'] } -# -# handlers: -# - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml new file mode 100644 index 0000000000..6058f31d31 --- /dev/null +++ b/roles/openqa/dispatcher/tasks/main.yml @@ -0,0 +1,99 @@ +# Required vars with defaults +# - openqa_hostname +## string - hostname of openQA server to run jobs for +## default - localhost +# - openqa_triggers +## list - the timers to enable - i.e. which scheduled runs +## will actually be triggered by this dispatcher +## default - ['current', 'branched', 'rawhide', 'stable'] +# +# Optional vars +# - wikitcms_user +## string - FAS username for reporting results to wiki +# - wikitcms_password +## string - password for relval_user +# +# When both of the above are set, a wikitcms 'credentials' file will +# be created and result submission to the wiki will be enabled. +# +# NOTE: At present, as the scheduler code downloads the ISOs, it only +# really makes sense for openqa_server boxes to be their own +# openqa_dispatchers. However, this should be fixed in the scheduler +# code soon, so the roles are kept separate to give future flexibility. + +- name: Install required packages + dnf: name={{ item }} state=present + with_items: + - fedfind + - python-wikitcms + - python-requests + - python-setuptools + - python-six + tags: + - packages + +- name: Check out openQA-python-client + git: + repo: https://github.com/os-autoinst/openQA-python-client.git + dest: /root/openQA-python-client + register: gitclient + +- name: Check if python-client has ever been installed + command: "python -c 'import openqa_client'" + register: instclient + changed_when: "1 != 1" + failed_when: "1 != 1" + +- name: Install openQA-python-client + command: "python setup.py install" + args: + chdir: /root/openQA-python-client + when: "gitclient|changed or instclient.rc > 0" + +- name: Check out openqa_fedora_tools + git: + repo: https://bitbucket.org/rajcze/openqa_fedora_tools + dest: /root/openqa_fedora_tools + register: gittools + +- name: Check if openqa_fedora_tools has ever been installed + stat: path=/usr/bin/fedora-openqa-schedule + register: insttools + changed_when: "1 != 1" + failed_when: "1 != 1" + +- name: Install openqa_fedora_tools + command: "python setup.py install" + args: + chdir: /root/openqa_fedora_tools + when: "gittools|changed or not insttools.stat.exists" + +- name: Reload systemd config to pick up installed units + command: "systemctl daemon-reload" + when: "gittools|changed or not insttools.stat.exists" + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600 + tags: + - config + +- name: Create fedora-openqa-schedule config directory + file: path=/etc/fedora-qa state=directory owner=root group=root mode=0700 + +- name: Write schedule.conf + template: src=schedule.conf.j2 dest=/etc/fedora-qa/schedule.conf owner=root group=root mode=0600 + tags: + - config + +- name: Create /root/.fedora (credentials files location) + file: path=/root/.fedora state=directory owner=root group=root mode=0700 + +- name: Write wikitcms credentials file + template: src=credentials.j2 dest=/root/.fedora/credentials owner=root group=root mode=0600 + when: "wikitcms_user is defined and wikitcms_password is defined" + tags: + - config + +- name: Enable and start timers + service: name=openqa-trigger-{{ item }}.timer enabled=yes state=started + with_items: "{{ openqa_triggers }}" diff --git a/roles/openqa/dispatcher/templates/client.conf.j2 b/roles/openqa/dispatcher/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/dispatcher/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/dispatcher/templates/credentials.j2 b/roles/openqa/dispatcher/templates/credentials.j2 new file mode 100644 index 0000000000..b460deaf6e --- /dev/null +++ b/roles/openqa/dispatcher/templates/credentials.j2 @@ -0,0 +1 @@ +{{ wikitcms_user }} {{ wikitcms_password }} diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2 new file mode 100644 index 0000000000..5c0c11dea2 --- /dev/null +++ b/roles/openqa/dispatcher/templates/schedule.conf.j2 @@ -0,0 +1,6 @@ +[report] +{% if wikitcms_user is defined and wikitcms_password is defined %} +submit: true +{% else %} +submit: false +{% endif %} diff --git a/roles/openqa/dispatcher/vars/main.yml b/roles/openqa/dispatcher/vars/main.yml new file mode 100644 index 0000000000..9284861122 --- /dev/null +++ b/roles/openqa/dispatcher/vars/main.yml @@ -0,0 +1,6 @@ +openqa_hostname: localhost +openqa_triggers: + - current + - branched + - rawhide + - stable diff --git a/roles/openqa/server/files/exports b/roles/openqa/server/files/exports new file mode 100644 index 0000000000..6243cf3a67 --- /dev/null +++ b/roles/openqa/server/files/exports @@ -0,0 +1 @@ +/var/lib/openqa/share *(ro,insecure,all_squash) diff --git a/roles/openqa/server/files/openqa.ini b/roles/openqa/server/files/openqa.ini new file mode 100644 index 0000000000..58b14fe906 --- /dev/null +++ b/roles/openqa/server/files/openqa.ini @@ -0,0 +1,12 @@ +[global] +branding = plain + +[auth] +method=OpenID + +[logging] +level=info + +[openid] +provider = https://id.fedoraproject.org/ +httpsonly = 1 diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml new file mode 100644 index 0000000000..4096d3158f --- /dev/null +++ b/roles/openqa/server/tasks/main.yml @@ -0,0 +1,167 @@ +# Required vars +# - openqa_email +## string - Email address of admin user +# - openqa_nickname +## string - Short name of admin user (shown in the web UI for e.g.) +# - openqa_fullname +## string - Full name of admin user +# - openqa_key +# - openqa_secret +## string - MUST be 16-character hexadecimals, and are secrets +# openqa_userid +## string - User ID of admin user: for Fedora should be a Fedora openID URL, +## http://fasname.id.fedoraproject.org + +# Required vars with defaults +# - external_hostname +## string - The public hostname for the server (will be used as ServerName) +## default - ansible_nodename + +# Optional vars +# - openqa_dbname +## string - The name of the database to use +# - openqa_dbhost +## string - The hostname of the database server +# - openqa_dbuser +## string - The database username +# - openqa_dbpassword +## string - The database password +# +# If openqa_dbhost is set, the others must be too, and the server will be +# configured to use a pgsql database accordingly. If openqa_dbhost is not +# set, the server will use a local SQLite database and the other values +# are ignored. + +- name: Ensure DNF COPR plugin is available + dnf: pkg="dnf-command(copr)" state=present + tags: + - packages + +- name: Install openQA repo if needed + command: "dnf -y copr enable adamwill/openQA" + args: + creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo + tags: + - config + +- name: Install required packages + dnf: name={{ item }} state=present enablerepo=adamwill-openQA + with_items: + - libselinux-python + - openqa + - git + - libselinux-utils + - libsemanage-python + - nfs-utils + - perl(Class::DBI::Pg) + - perl(DateTime::Format::Pg) + - libguestfs-tools-c + tags: + - packages + +- name: Check out the tests + git: + repo: https://bitbucket.org/rajcze/openqa_fedora + dest: /var/lib/openqa/share/tests/fedora + +- name: Have tests owned by geekotest + file: path=/var/lib/openqa/share/tests/fedora owner=geekotest recurse=yes + +- name: Check out openqa_fedora_tools + git: + repo: https://bitbucket.org/rajcze/openqa_fedora_tools + dest: /root/openqa_fedora_tools + +- name: Create asset directories + file: path={{ item }} state=directory owner=geekotest group=root mode=0755 + with_items: + - /var/lib/openqa/share/factory/iso + - /var/lib/openqa/share/factory/hdd + - /var/lib/openqa/share/factory/repo + +# NOTE: this is very hacky, but we can't do much better with the current +# disk creation script, I will try and make it better. We'll have to bump +# this hardcoded release number every so often. +- name: Create hard disk images (this may take a long time!) + command: "/root/openqa_fedora_tools/tools/createhdds.sh 23" + args: + creates: /var/lib/openqa/share/factory/hdd/disk_full.img + chdir: /var/lib/openqa/share/factory/hdd/ + +- name: Create exports file + copy: src=exports dest=/etc/exports.d/openqa.exports owner=root group=root mode=0644 + tags: + - config + +- name: Enable and start NFS server + service: name=nfs-server enabled=yes state=started + +- name: Set up Apache config + template: src=openqa.conf.httpd.j2 dest=/etc/httpd/conf.d/openqa.conf owner=root group=root mode=0644 + tags: + - config + +- name: OpenQA config + copy: src=openqa.ini dest=/etc/openqa/openqa.ini owner=geekotest group=root mode=0640 + tags: + - config + +- name: Create database + delegate_to: "{{ openqa_dbhost }}" + sudo_user: postgres + sudo: true + action: postgresql_db db={{ openqa_dbname }} + when: "openqa_dbhost is defined" + +- name: Ensure db user has access to database + delegate_to: "{{ openqa_dbhost }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ openqa_dbname }} user={{ openqa_dbuser }} password={{ openqa_dbpassword }} role_attr_flags=NOSUPERUSER + when: "openqa_dbhost is defined" + +- name: Database config + template: src=database.ini.pgsql.j2 dest=/etc/openqa/database.ini owner=geekotest group=root mode=0640 + when: "openqa_dbhost is defined" + tags: + - config + +- name: Initialize database + command: "/usr/share/openqa/script/initdb --user geekotest --init_database" + register: initdb + changed_when: "initdb.rc == 0" + failed_when: "(initdb.rc > 0) and (initdb.stderr is not defined or initdb.stderr.find('already exists') == -1)" + +- name: Enable and start services + service: name={{ item }} enabled=yes state=started + register: services + with_items: + - openqa-scheduler + - openqa-webui + - openqa-websockets + - openqa-gru + +# This is using a big hammer until #1277312 is resolved +- name: Allow Apache to connect to openQA + seboolean: name=httpd_can_network_connect state=yes persistent=yes + +- name: Wait for openQA to be fully started + pause: seconds=5 + when: services|changed + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600 + tags: + - config + +- name: Create admin user + command: "/var/lib/openqa/script/create_admin --email {{ openqa_email }} --nickname {{ openqa_nickname }} --fullname '{{ openqa_fullname }}' --key {{ openqa_key }} --secret {{ openqa_secret }} {{ openqa_userid }}" + register: admin + changed_when: "admin.rc == 0" + failed_when: "(admin.rc > 0) and (admin.stderr is not defined or admin.stderr.find('already exists') == -1)" + +- name: Load tests + shell: "/var/lib/openqa/share/tests/fedora/templates | grep 'added => [1-9]'" + register: templates + changed_when: "templates.rc == 0" + failed_when: "1 != 1" diff --git a/roles/openqa/server/templates/client.conf.j2 b/roles/openqa/server/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/server/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/server/templates/database.ini.pgsql.j2 b/roles/openqa/server/templates/database.ini.pgsql.j2 new file mode 100644 index 0000000000..05712e2c80 --- /dev/null +++ b/roles/openqa/server/templates/database.ini.pgsql.j2 @@ -0,0 +1,9 @@ +[test] +dsn = dbi:SQLite:dbname=:memory: +on_connect_call = use_foreign_keys +on_connect_do = PRAGMA synchronous = OFF + +[production] +dsn = dbi:Pg:dbname={{ openqa_dbname }};host={{ openqa_dbhost }} +user = {{ openqa_dbuser }} +password = {{ openqa_dbpassword }} diff --git a/roles/openqa/server/templates/openqa.conf.httpd.j2 b/roles/openqa/server/templates/openqa.conf.httpd.j2 new file mode 100644 index 0000000000..dd8d9e6195 --- /dev/null +++ b/roles/openqa/server/templates/openqa.conf.httpd.j2 @@ -0,0 +1,4 @@ + + ServerName {{ external_hostname|default(ansible_nodename) }} + Include conf.d/openqa-common.inc + diff --git a/roles/openqa/worker/tasks/main.yml b/roles/openqa/worker/tasks/main.yml new file mode 100644 index 0000000000..70c4c70ed9 --- /dev/null +++ b/roles/openqa/worker/tasks/main.yml @@ -0,0 +1,45 @@ +# Required vars +# - openqa_workers +## integer - number of worker instances to create/run + +# Required vars with defaults +# - openqa_hostname +## string - hostname of openQA server to run jobs for +## default - localhost + +- name: Ensure DNF COPR plugin is available + dnf: pkg="dnf-command(copr)" state=present + tags: + - packages + +- name: Install openQA repo if needed + command: "dnf -y copr enable adamwill/openQA" + args: + creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo + tags: + - config + +- name: Install packages + dnf: name={{ item }} state=present enablerepo=adamwill-openQA + with_items: + - openqa-worker + - libselinux-python + tags: + - packages + +- include: nfs-client.yml + when: openqa_hostname is defined and openqa_hostname != "localhost" + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf owner=_openqa-worker group=root mode=0600 + tags: + - config + +- name: openQA worker config + template: src=workers.ini.j2 dest=/etc/openqa/workers.ini owner=_openqa-worker group=root mode=0644 + tags: + - config + +- name: Worker services + service: name=openqa-worker@{{ item }} enabled=yes state=started + with_sequence: "count={{ openqa_workers }}" diff --git a/roles/openqa/worker/tasks/nfs-client.yml b/roles/openqa/worker/tasks/nfs-client.yml new file mode 100644 index 0000000000..2eedd0ec3f --- /dev/null +++ b/roles/openqa/worker/tasks/nfs-client.yml @@ -0,0 +1,24 @@ +# Required vars +# - openqa_hostname +## string - hostname of the openQA server (we assume it is hosting the NFS mount) + +- name: Install NFS client + dnf: name=nfs-utils state=present + tags: + - packages + +# We don't check ownership as, after mounting, it's owned by whatever the +# UID of geekotest is on the server +- name: Ensure mount target exists + file: path=/var/lib/openqa/share state=directory mode=0755 + +- name: Create mount unit + template: src=var-lib-openqa-share.mount.j2 dest=/etc/systemd/system/var-lib-openqa-share.mount owner=root group=root mode=0644 + tags: + - config + +- name: Enable and start mount + service: name={{ item }} enabled=yes state=started + with_items: + - var-lib-openqa-share.mount + - remote-fs.target diff --git a/roles/openqa/worker/templates/client.conf.j2 b/roles/openqa/worker/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/worker/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 b/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 new file mode 100644 index 0000000000..3ec823f460 --- /dev/null +++ b/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=openQA shared assets +ConditionPathExists=/var/lib/openqa/share + +[Mount] +What={{ openqa_hostname }}:/var/lib/openqa/share +Where=/var/lib/openqa/share +Type=nfs + +[Install] +WantedBy = remote-fs.target diff --git a/roles/openqa/worker/templates/workers.ini.j2 b/roles/openqa/worker/templates/workers.ini.j2 new file mode 100644 index 0000000000..b9665bc8fc --- /dev/null +++ b/roles/openqa/worker/templates/workers.ini.j2 @@ -0,0 +1,2 @@ +[global] +HOST = http://{{ openqa_hostname|default('localhost') }} diff --git a/roles/openqa/worker/vars/main.yml b/roles/openqa/worker/vars/main.yml new file mode 100644 index 0000000000..42aacce60c --- /dev/null +++ b/roles/openqa/worker/vars/main.yml @@ -0,0 +1 @@ +openqa_hostname: localhost