diff --git a/inventory/group_vars/koji b/inventory/group_vars/koji index 07e948a5aa..54da467cdb 100644 --- a/inventory/group_vars/koji +++ b/inventory/group_vars/koji @@ -50,5 +50,3 @@ virt_install_command: virt-install -n {{ inventory_hostname }} -r {{ mem_size }} ip={{ eth1_ip }}:::{{ nm }}:{{ inventory_hostname }}-nfs:eth1:none" --network=bridge=br0,model=virtio --network=bridge=br1,model=virtio --autostart --noautoconsole - -sudoers: "{{ private }}/files/sudo/arm-releng-sudoers" diff --git a/inventory/group_vars/openqa b/inventory/group_vars/openqa index e69de29bb2..81c5faae8d 100644 --- a/inventory/group_vars/openqa +++ b/inventory/group_vars/openqa @@ -0,0 +1,18 @@ +external_hostname: openqa.fedoraproject.org + +openqa_hostname: localhost +openqa_email: adamwill@fedoraproject.org +openqa_nickname: adamwill +openqa_fullname: Adam Williamson +openqa_userid: http://adamwill.id.fedoraproject.org/ + +openqa_dbname: openqa +openqa_dbhost: db-qa01.qa.fedoraproject.org +openqa_dbuser: openqa + +wikitcms_user: coconut + +openqa_workers: 3 + +# NFS +tcp_ports: 2049 diff --git a/inventory/group_vars/openqa-stg b/inventory/group_vars/openqa-stg index e69de29bb2..7620270180 100644 --- a/inventory/group_vars/openqa-stg +++ b/inventory/group_vars/openqa-stg @@ -0,0 +1,19 @@ +external_hostname: openqa-stg.fedoraproject.org + +openqa_hostname: localhost +openqa_email: adamwill@fedoraproject.org +openqa_nickname: adamwill +openqa_fullname: Adam Williamson +openqa_userid: http://adamwill.id.fedoraproject.org/ + +openqa_dbname: openqa-stg +openqa_dbhost: db-qa01.qa.fedoraproject.org +openqa_dbuser: openqa + +openqa_workers: 3 + +# For now let's not run any scheduled jobs on stg +openqa_triggers: [] + +# NFS +tcp_ports: 2049 diff --git a/inventory/group_vars/openqa-stg-workers b/inventory/group_vars/openqa-stg-workers new file mode 100644 index 0000000000..0f56cd18b7 --- /dev/null +++ b/inventory/group_vars/openqa-stg-workers @@ -0,0 +1,2 @@ +openqa_workers: 4 +openqa_hostname: {{ groups['openqa-stg'][0] }} diff --git a/inventory/group_vars/openqa-workers b/inventory/group_vars/openqa-workers new file mode 100644 index 0000000000..9f8c683642 --- /dev/null +++ b/inventory/group_vars/openqa-workers @@ -0,0 +1,2 @@ +openqa_workers: 4 +openqa_hostname: {{ groups['openqa'][0] }} diff --git a/inventory/group_vars/qa-stg b/inventory/group_vars/qa-stg index 1c1a8d6efd..e2f392e75e 100644 --- a/inventory/group_vars/qa-stg +++ b/inventory/group_vars/qa-stg @@ -63,7 +63,6 @@ buildmaster_dir: /home/buildmaster/master buildslave_dir: /home/buildslave/slave buildslave_poll_interval: 1800 buildmaster_home: /home/buildmaster -buildmaster_dir: /home/buildmaster/master buildmaster_user: buildmaster # build details diff --git a/inventory/group_vars/qadevel b/inventory/group_vars/qadevel index 7f6f21259a..41c03d0ef3 100644 --- a/inventory/group_vars/qadevel +++ b/inventory/group_vars/qadevel @@ -34,7 +34,6 @@ buildmaster_dir: /home/buildmaster/master buildslave_dir: /home/buildslave/slave buildslave_poll_interval: 1800 buildmaster_home: /home/buildmaster -buildmaster_dir: /home/buildmaster/master buildmaster_user: buildmaster external_hostname: qadevel.cloud.fedoraproject.org diff --git a/inventory/group_vars/taskotron-prod b/inventory/group_vars/taskotron-prod index 7c1e5c7cb9..2a94e23d65 100644 --- a/inventory/group_vars/taskotron-prod +++ b/inventory/group_vars/taskotron-prod @@ -16,7 +16,6 @@ cgit_root_title: "Taskotron Git Mirror" buildmaster_dir: /home/buildmaster/master buildslave_dir: /home/buildslave/slave buildmaster_home: /home/buildmaster -buildmaster_dir: /home/buildmaster/master buildmaster_user: buildmaster external_hostname: taskotron.fedoraproject.org resultsdb_url: http://resultsdb01.qa.fedoraproject.org/resultsdb_api/api/v1.0 diff --git a/inventory/group_vars/taskotron-prod-clients b/inventory/group_vars/taskotron-prod-clients index 37298adfd3..fc3fc5c6cb 100644 --- a/inventory/group_vars/taskotron-prod-clients +++ b/inventory/group_vars/taskotron-prod-clients @@ -20,7 +20,6 @@ freezes: true slave_user: buildslave slave_home: /home/buildslave/ slave_dir: /home/buildslave/slave -slave_user: buildslave buildmaster: 10.5.124.206 buildslave_port: 9989 taskotron_admin_email: taskotron-admin-members@fedoraproject.org diff --git a/inventory/group_vars/taskotron-stg-clients b/inventory/group_vars/taskotron-stg-clients index bfae6784bd..ee7b55319b 100644 --- a/inventory/group_vars/taskotron-stg-clients +++ b/inventory/group_vars/taskotron-stg-clients @@ -24,7 +24,6 @@ extra_enablerepos: '' slave_user: buildslave slave_home: /home/buildslave/ slave_dir: /home/buildslave/slave -slave_user: buildslave buildmaster: 10.5.124.232 buildslave_port: 9989 taskotron_admin_email: taskotron-admin-members@fedoraproject.org diff --git a/inventory/host_vars/autocloud-backend-vbox.phx2.fedoraproject.org b/inventory/host_vars/autocloud-backend-vbox.phx2.fedoraproject.org index 438b31f398..9c66838eba 100644 --- a/inventory/host_vars/autocloud-backend-vbox.phx2.fedoraproject.org +++ b/inventory/host_vars/autocloud-backend-vbox.phx2.fedoraproject.org @@ -10,7 +10,6 @@ nfs_mount_opts: "rw,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3" # general configs nrpe_procs_warn: 900 nrpe_procs_crit: 1000 -datacenter: phx2 nm: 255.255.255.0 gw: 10.5.126.254 dns: 10.5.126.21 diff --git a/inventory/host_vars/copr-fe.cloud.fedoraproject.org b/inventory/host_vars/copr-fe.cloud.fedoraproject.org index fc32e34789..d48ae346c9 100644 --- a/inventory/host_vars/copr-fe.cloud.fedoraproject.org +++ b/inventory/host_vars/copr-fe.cloud.fedoraproject.org @@ -11,7 +11,6 @@ hostbase: copr-fe- public_ip: 209.132.184.54 root_auth_users: ryanlerch pingou msuchy sgallagh nb asamalik vgologuz description: copr frontend server - prod instance -volumes: ['-d /dev/vdb vol-0000000f'] tcp_ports: [22, 80, 443] volumes: [ {volume_id: '8f790db7-8294-4d2b-8bae-7af5961ce0f8', device: '/dev/vdc'} ] inventory_tenant: persistent diff --git a/inventory/host_vars/jenkins.fedorainfracloud.org b/inventory/host_vars/jenkins.fedorainfracloud.org index e470bf4ae3..61f7128a6b 100644 --- a/inventory/host_vars/jenkins.fedorainfracloud.org +++ b/inventory/host_vars/jenkins.fedorainfracloud.org @@ -4,7 +4,7 @@ instance_type: m1.small keypair: fedora-admin-20130801 security_group: ssh-anywhere-persistent,web-80-anywhere-persistent,default,all-icmp-persistent zone: nova -tcp_ports: [22, 80, 443] +tcp_ports: [22, 80, 443, 8080] inventory_tenant: persistent inventory_instance_name: jenkins @@ -21,8 +21,6 @@ cloud_networks: # persistent-net - net-id: "67b77354-39a4-43de-b007-bb813ac5c35f" -tcp_ports: [ 8080 ] - custom_nat_rules: [ # Redirect port 80 to 8080, which is used by jenkins '-A PREROUTING -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080', diff --git a/inventory/host_vars/mm-crawler01.phx2.fedoraproject.org b/inventory/host_vars/mm-crawler01.phx2.fedoraproject.org index 0f364b8ee6..6547fa7c89 100644 --- a/inventory/host_vars/mm-crawler01.phx2.fedoraproject.org +++ b/inventory/host_vars/mm-crawler01.phx2.fedoraproject.org @@ -1,6 +1,6 @@ --- lvm_size: 20000 -mem_size: 32768 +mem_size: 40960 num_cpus: 4 nm: 255.255.255.0 gw: 10.5.126.254 diff --git a/inventory/host_vars/mm-crawler02.phx2.fedoraproject.org b/inventory/host_vars/mm-crawler02.phx2.fedoraproject.org index 7a27f7f13a..804afa116c 100644 --- a/inventory/host_vars/mm-crawler02.phx2.fedoraproject.org +++ b/inventory/host_vars/mm-crawler02.phx2.fedoraproject.org @@ -1,6 +1,6 @@ --- lvm_size: 20000 -mem_size: 32768 +mem_size: 40960 num_cpus: 4 nm: 255.255.255.0 gw: 10.5.126.254 diff --git a/inventory/host_vars/people01.fedoraproject.org b/inventory/host_vars/people01.fedoraproject.org index e4cc0ce312..53d5baeed0 100644 --- a/inventory/host_vars/people01.fedoraproject.org +++ b/inventory/host_vars/people01.fedoraproject.org @@ -1,6 +1,5 @@ --- freezes: false -datacenter: ibiblio #host_backup_targets: ['/srv/web'] nm: 255.255.255.128 diff --git a/inventory/host_vars/proxy07.fedoraproject.org b/inventory/host_vars/proxy07.fedoraproject.org index 4861605072..8e391f4681 100644 --- a/inventory/host_vars/proxy07.fedoraproject.org +++ b/inventory/host_vars/proxy07.fedoraproject.org @@ -8,8 +8,6 @@ eth0_ip: 213.175.193.206 vmhost: bodhost01.fedoraproject.org datacenter: bodhost -postfix_group: vpn - ks_url: http://209.132.181.6/repo/rhel/ks/kvm-rhel-7-ext ks_repo: http://209.132.181.6/repo/rhel/RHEL7-x86_64/ diff --git a/inventory/host_vars/testdays.fedorainfracloud.org b/inventory/host_vars/testdays.fedorainfracloud.org index 0e14323d83..c62328ec77 100644 --- a/inventory/host_vars/testdays.fedorainfracloud.org +++ b/inventory/host_vars/testdays.fedorainfracloud.org @@ -46,7 +46,6 @@ resultsdb_db_host_machine: "{{ public_ip }}" resultsdb_db_host: 127.0.0.1 resultsdb_db_port: 5432 resultsdb_endpoint: 'resultsdb_api' -resultsdb_fe_endpoint: 'resultsdb' resultsdb_db_name: resultsdb_testdays resultsdb_db_user: "{{ testdays_testdays_db_user }}" @@ -56,9 +55,6 @@ resultsdb_secret_key: "{{ testdays_resultsdb_secretkey }}" allowed_hosts: - 127.0.0.1 -resultsdb_endpoint: "resultsdb-api" - - ############################################################ # resultsdb_frontend ############################################################ diff --git a/playbooks/groups/buildhw.yml b/playbooks/groups/buildhw.yml index 2b3f703076..ea4e6d43e8 100644 --- a/playbooks/groups/buildhw.yml +++ b/playbooks/groups/buildhw.yml @@ -12,6 +12,9 @@ - "/srv/private/ansible/vars.yml" - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + pre_tasks: + - include: "{{ tasks }}/yumrepos.yml" + roles: - base - { role: nfs/client, when: inventory_hostname.startswith('build') , mnt_dir: '/mnt/fedora_koji', nfs_src_dir: 'fedora_koji' } @@ -30,7 +33,6 @@ when: not inventory_hostname.startswith('bkernel') - include: "{{ tasks }}/motd.yml" when: not inventory_hostname.startswith('bkernel') - - include: "{{ tasks }}/yumrepos.yml" handlers: - include: "{{ handlers }}/restart_services.yml" diff --git a/playbooks/groups/noc.yml b/playbooks/groups/noc.yml index f444407912..115792e7f2 100644 --- a/playbooks/groups/noc.yml +++ b/playbooks/groups/noc.yml @@ -42,7 +42,7 @@ - /srv/web/infra/ansible/vars/global.yml - "/srv/private/ansible/vars.yml" - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - /srv/web/infra/ansible/vars/nagios.{{ inventory_hostname_short }}.yml + - "/srv/web/infra/ansible/vars/nagios.{{ inventory_hostname_short }}.yml" handlers: - include: "{{ handlers }}/restart_services.yml" diff --git a/playbooks/groups/openqa-workers.yml b/playbooks/groups/openqa-workers.yml index 6425876202..5be3f43346 100644 --- a/playbooks/groups/openqa-workers.yml +++ b/playbooks/groups/openqa-workers.yml @@ -1,30 +1,23 @@ -# create a new openqa worker server system -# NOTE: should be used with --limit most of the time -# NOTE: most of these vars_path come from group_vars/backup_server or from hostvars -# This has an extra role that configures the virthost to be used with beaker for -# virtual machine clients - -- name: basic configuration +- name: configure openQA workers hosts: openqa-workers:openqa-stg-workers user: root gather_facts: True - vars_files: + vars_files: - /srv/web/infra/ansible/vars/global.yml - "/srv/private/ansible/vars.yml" - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: - - base - - rkhunter - - { role: denyhosts, when: ansible_distribution_major_version|int != 7 } - - nagios_client - - hosts - - fas_client - - collectd/base - - { role: iscsi_client, when: datacenter == "phx2" } - - sudo - - { role: openvpn/client, when: datacenter != "phx2" } + - { role: base, tags: ['base'] } + - { role: rkhunter, tags: ['rkhunter'] } + - { role: nagios_client, tags: ['nagios_client'] } + - { role: hosts, tags: ['hosts']} + - { role: fas_client, tags: ['fas_client'] } + - { role: collectd/base, tags: ['collectd_base'] } + - { role: sudo, tags: ['sudo'] } + - { role: openqa_worker, tags: ['openqa_worker'] } + - apache tasks: - include: "{{ tasks }}/yumrepos.yml" @@ -32,20 +25,4 @@ - include: "{{ tasks }}/motd.yml" handlers: - - include: "{{ handlers }}/restart_services.yml" - -#- name: configure openqa workers -# hosts: openqa-workers:openqa-stg-workers -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - { role: openqa/something, tags: ['something'] } -# -# handlers: -# - include: "{{ handlers }}/restart_services.yml" + - include: "{{ handlers }}/restart_services.yml" diff --git a/playbooks/groups/openqa.yml b/playbooks/groups/openqa.yml index 158be787b1..3bc34e5ae6 100644 --- a/playbooks/groups/openqa.yml +++ b/playbooks/groups/openqa.yml @@ -1,11 +1,4 @@ ---- -# create a new taskotron staging server -# NOTE: make sure there is room/space for this server on the vmhost -# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars - -- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=openqa:openqa-stg" - -- name: make the box be real +- name: configure openQA hosts: openqa:openqa-stg user: root gather_facts: True @@ -22,31 +15,16 @@ - { role: hosts, tags: ['hosts']} - { role: fas_client, tags: ['fas_client'] } - { role: collectd/base, tags: ['collectd_base'] } -# - { role: yum-cron, tags: ['yumcron'] } - { role: sudo, tags: ['sudo'] } + - { role: openqa/server, tags: ['openqa_server'] } + - { role: openqa/worker, tags: ['openqa_worker'] } + - { role: openqa/dispatcher, tags: ['openqa_dispatcher'] } - apache tasks: - # this is how you include other task lists - include: "{{ tasks }}/yumrepos.yml" - include: "{{ tasks }}/2fa_client.yml" - include: "{{ tasks }}/motd.yml" handlers: - include: "{{ handlers }}/restart_services.yml" - -#- name: configure openqa -# hosts: openqa:openqa-stg -# user: root -# gather_facts: True -# -# vars_files: -# - /srv/web/infra/ansible/vars/global.yml -# - "/srv/private/ansible/vars.yml" -# - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml -# -# roles: -# - { role: openqa/something, tags: ['something'] } -# -# handlers: -# - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/mirrormanager/crawler/files/crawler.cron b/roles/mirrormanager/crawler/files/crawler.cron index a3691d7732..2b68c76bc5 100644 --- a/roles/mirrormanager/crawler/files/crawler.cron +++ b/roles/mirrormanager/crawler/files/crawler.cron @@ -1,8 +1,7 @@ # run the crawler twice a day # logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default -# 27 threads with 32GB of RAM seems to work so far # # [ "`hostname -s`" == "mm-crawler02" ] && sleep 2h is used to start the crawl # later on the second crawler to reduce the number of parallel accesses to # the database -0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 2h; /usr/bin/mm2_crawler --timeout-minutes 180 --threads 23 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 +0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 2h; /usr/bin/mm2_crawler --timeout-minutes 180 --threads 20 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1 diff --git a/roles/mirrormanager/mirrorlist2/files/mirrorlist-server.service b/roles/mirrormanager/mirrorlist2/files/mirrorlist-server.service new file mode 100644 index 0000000000..f4932e18bd --- /dev/null +++ b/roles/mirrormanager/mirrorlist2/files/mirrorlist-server.service @@ -0,0 +1,5 @@ +.include /usr/lib/systemd/system/mirrorlist-server.service +[Service] +# systemd needs an empty ExecStart= to be able to overwrite ExecStart= +ExecStart= +ExecStart=/usr/bin/python2 /usr/share/mirrormanager2/mirrorlist_server.py --debug -l /var/log/mirrormanager/mirrorlist.log diff --git a/roles/mirrormanager/mirrorlist2/handlers/main.yml b/roles/mirrormanager/mirrorlist2/handlers/main.yml new file mode 100644 index 0000000000..3d8bee041e --- /dev/null +++ b/roles/mirrormanager/mirrorlist2/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reload systemd service files + command: systemctl daemon-reload diff --git a/roles/mirrormanager/mirrorlist2/tasks/main.yml b/roles/mirrormanager/mirrorlist2/tasks/main.yml index e0dee54162..2cf504e321 100644 --- a/roles/mirrormanager/mirrorlist2/tasks/main.yml +++ b/roles/mirrormanager/mirrorlist2/tasks/main.yml @@ -80,5 +80,16 @@ owner=root group=root mode=0755 tags: - hotfix + +- name: make a /var/log/mirrormanager dir for logs + file: dest=/var/log/mirrormanager/ state=directory owner=mirrormanager group=mirrormanager mode=0755 + tags: + - mirrorlist2 + +- name: copy systemd service file in for mirrorlist-server + copy: src=mirrorlist-server.service dest=/etc/systemd/system/mirrorlist-server.service notify: + - reload systemd service files - restart mirrorlist-server + tags: + - mirrorlist2 diff --git a/roles/openqa/dispatcher/tasks/main.yml b/roles/openqa/dispatcher/tasks/main.yml new file mode 100644 index 0000000000..a9bb067862 --- /dev/null +++ b/roles/openqa/dispatcher/tasks/main.yml @@ -0,0 +1,99 @@ +# Required vars with defaults +# - openqa_hostname +## string - hostname of openQA server to run jobs for +## default - localhost +# - openqa_triggers +## list - the timers to enable - i.e. which scheduled runs +## will actually be triggered by this dispatcher +## default - ['current', 'branched', 'rawhide', 'stable'] +# +# Optional vars +# - wikitcms_user +## string - FAS username for reporting results to wiki +# - wikitcms_password +## string - password for relval_user +# +# When both of the above are set, a wikitcms 'credentials' file will +# be created and result submission to the wiki will be enabled. +# +# NOTE: At present, as the scheduler code downloads the ISOs, it only +# really makes sense for openqa_server boxes to be their own +# openqa_dispatchers. However, this should be fixed in the scheduler +# code soon, so the roles are kept separate to give future flexibility. + +- name: Install required packages + dnf: name={{ item }} state=present + with_items: + - fedfind + - python-wikitcms + - python-requests + - python-setuptools + - python-six + tags: + - packages + +- name: Check out openQA-python-client + git: + repo: https://github.com/os-autoinst/openQA-python-client.git + dest: /root/openQA-python-client + register: gitclient + +- name: Check if python-client has ever been installed + command: "python -c 'import openqa_client'" + register: instclient + changed_when: "1 != 1" + failed_when: "1 != 1" + +- name: Install openQA-python-client + command: "python setup.py install" + args: + chdir: /root/openQA-python-client + when: "gitclient|changed or instclient.rc > 0" + +- name: Check out openqa_fedora_tools + git: + repo: https://bitbucket.org/rajcze/openqa_fedora_tools + dest: /root/openqa_fedora_tools + register: gittools + +- name: Check if openqa_fedora_tools has ever been installed + stat: path=/usr/bin/fedora-openqa-schedule + register: insttools + changed_when: "1 != 1" + failed_when: "1 != 1" + +- name: Install openqa_fedora_tools + command: "python setup.py install" + args: + chdir: /root/openqa_fedora_tools + when: "gittools|changed or not insttools.stat.exists" + +- name: Reload systemd config to pick up installed units + command: "systemctl daemon-reload" + when: "gittools|changed or not insttools.stat.exists" + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600 + tags: + - config + +- name: Create fedora-openqa-schedule config directory + file: path=/etc/fedora-qa state=directory owner=root group=root mode=0700 + +- name: Write schedule.conf + template: src=schedule.conf.j2 dest=/etc/fedora-qa/schedule.conf owner=root group=root mode=0600 + tags: + - config + +- name: Create /root/.fedora (credentials files location) + file: path=/root/.fedora state=directory owner=root group=root mode=0700 + +- name: Write wikitcms credentials file + template: src=credentials.j2 dest=/root/.fedora/credentials owner=root group=root mode=0600 + when: "wikitcms_user is defined and wikitcms_password is defined" + tags: + - config + +- name: Enable and start timers + service: name=openqa-trigger-{{ item }}.timer enabled=yes state=started + with_items: "{{ openqa_triggers }}" diff --git a/roles/openqa/dispatcher/templates/client.conf.j2 b/roles/openqa/dispatcher/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/dispatcher/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/dispatcher/templates/credentials.j2 b/roles/openqa/dispatcher/templates/credentials.j2 new file mode 100644 index 0000000000..b460deaf6e --- /dev/null +++ b/roles/openqa/dispatcher/templates/credentials.j2 @@ -0,0 +1 @@ +{{ wikitcms_user }} {{ wikitcms_password }} diff --git a/roles/openqa/dispatcher/templates/schedule.conf.j2 b/roles/openqa/dispatcher/templates/schedule.conf.j2 new file mode 100644 index 0000000000..5c0c11dea2 --- /dev/null +++ b/roles/openqa/dispatcher/templates/schedule.conf.j2 @@ -0,0 +1,6 @@ +[report] +{% if wikitcms_user is defined and wikitcms_password is defined %} +submit: true +{% else %} +submit: false +{% endif %} diff --git a/roles/openqa/dispatcher/vars/main.yml b/roles/openqa/dispatcher/vars/main.yml new file mode 100644 index 0000000000..9284861122 --- /dev/null +++ b/roles/openqa/dispatcher/vars/main.yml @@ -0,0 +1,6 @@ +openqa_hostname: localhost +openqa_triggers: + - current + - branched + - rawhide + - stable diff --git a/roles/openqa/server/files/exports b/roles/openqa/server/files/exports new file mode 100644 index 0000000000..6243cf3a67 --- /dev/null +++ b/roles/openqa/server/files/exports @@ -0,0 +1 @@ +/var/lib/openqa/share *(ro,insecure,all_squash) diff --git a/roles/openqa/server/files/openqa.ini b/roles/openqa/server/files/openqa.ini new file mode 100644 index 0000000000..58b14fe906 --- /dev/null +++ b/roles/openqa/server/files/openqa.ini @@ -0,0 +1,12 @@ +[global] +branding = plain + +[auth] +method=OpenID + +[logging] +level=info + +[openid] +provider = https://id.fedoraproject.org/ +httpsonly = 1 diff --git a/roles/openqa/server/tasks/main.yml b/roles/openqa/server/tasks/main.yml new file mode 100644 index 0000000000..4096d3158f --- /dev/null +++ b/roles/openqa/server/tasks/main.yml @@ -0,0 +1,167 @@ +# Required vars +# - openqa_email +## string - Email address of admin user +# - openqa_nickname +## string - Short name of admin user (shown in the web UI for e.g.) +# - openqa_fullname +## string - Full name of admin user +# - openqa_key +# - openqa_secret +## string - MUST be 16-character hexadecimals, and are secrets +# openqa_userid +## string - User ID of admin user: for Fedora should be a Fedora openID URL, +## http://fasname.id.fedoraproject.org + +# Required vars with defaults +# - external_hostname +## string - The public hostname for the server (will be used as ServerName) +## default - ansible_nodename + +# Optional vars +# - openqa_dbname +## string - The name of the database to use +# - openqa_dbhost +## string - The hostname of the database server +# - openqa_dbuser +## string - The database username +# - openqa_dbpassword +## string - The database password +# +# If openqa_dbhost is set, the others must be too, and the server will be +# configured to use a pgsql database accordingly. If openqa_dbhost is not +# set, the server will use a local SQLite database and the other values +# are ignored. + +- name: Ensure DNF COPR plugin is available + dnf: pkg="dnf-command(copr)" state=present + tags: + - packages + +- name: Install openQA repo if needed + command: "dnf -y copr enable adamwill/openQA" + args: + creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo + tags: + - config + +- name: Install required packages + dnf: name={{ item }} state=present enablerepo=adamwill-openQA + with_items: + - libselinux-python + - openqa + - git + - libselinux-utils + - libsemanage-python + - nfs-utils + - perl(Class::DBI::Pg) + - perl(DateTime::Format::Pg) + - libguestfs-tools-c + tags: + - packages + +- name: Check out the tests + git: + repo: https://bitbucket.org/rajcze/openqa_fedora + dest: /var/lib/openqa/share/tests/fedora + +- name: Have tests owned by geekotest + file: path=/var/lib/openqa/share/tests/fedora owner=geekotest recurse=yes + +- name: Check out openqa_fedora_tools + git: + repo: https://bitbucket.org/rajcze/openqa_fedora_tools + dest: /root/openqa_fedora_tools + +- name: Create asset directories + file: path={{ item }} state=directory owner=geekotest group=root mode=0755 + with_items: + - /var/lib/openqa/share/factory/iso + - /var/lib/openqa/share/factory/hdd + - /var/lib/openqa/share/factory/repo + +# NOTE: this is very hacky, but we can't do much better with the current +# disk creation script, I will try and make it better. We'll have to bump +# this hardcoded release number every so often. +- name: Create hard disk images (this may take a long time!) + command: "/root/openqa_fedora_tools/tools/createhdds.sh 23" + args: + creates: /var/lib/openqa/share/factory/hdd/disk_full.img + chdir: /var/lib/openqa/share/factory/hdd/ + +- name: Create exports file + copy: src=exports dest=/etc/exports.d/openqa.exports owner=root group=root mode=0644 + tags: + - config + +- name: Enable and start NFS server + service: name=nfs-server enabled=yes state=started + +- name: Set up Apache config + template: src=openqa.conf.httpd.j2 dest=/etc/httpd/conf.d/openqa.conf owner=root group=root mode=0644 + tags: + - config + +- name: OpenQA config + copy: src=openqa.ini dest=/etc/openqa/openqa.ini owner=geekotest group=root mode=0640 + tags: + - config + +- name: Create database + delegate_to: "{{ openqa_dbhost }}" + sudo_user: postgres + sudo: true + action: postgresql_db db={{ openqa_dbname }} + when: "openqa_dbhost is defined" + +- name: Ensure db user has access to database + delegate_to: "{{ openqa_dbhost }}" + sudo_user: postgres + sudo: true + action: postgresql_user db={{ openqa_dbname }} user={{ openqa_dbuser }} password={{ openqa_dbpassword }} role_attr_flags=NOSUPERUSER + when: "openqa_dbhost is defined" + +- name: Database config + template: src=database.ini.pgsql.j2 dest=/etc/openqa/database.ini owner=geekotest group=root mode=0640 + when: "openqa_dbhost is defined" + tags: + - config + +- name: Initialize database + command: "/usr/share/openqa/script/initdb --user geekotest --init_database" + register: initdb + changed_when: "initdb.rc == 0" + failed_when: "(initdb.rc > 0) and (initdb.stderr is not defined or initdb.stderr.find('already exists') == -1)" + +- name: Enable and start services + service: name={{ item }} enabled=yes state=started + register: services + with_items: + - openqa-scheduler + - openqa-webui + - openqa-websockets + - openqa-gru + +# This is using a big hammer until #1277312 is resolved +- name: Allow Apache to connect to openQA + seboolean: name=httpd_can_network_connect state=yes persistent=yes + +- name: Wait for openQA to be fully started + pause: seconds=5 + when: services|changed + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf mode=0600 + tags: + - config + +- name: Create admin user + command: "/var/lib/openqa/script/create_admin --email {{ openqa_email }} --nickname {{ openqa_nickname }} --fullname '{{ openqa_fullname }}' --key {{ openqa_key }} --secret {{ openqa_secret }} {{ openqa_userid }}" + register: admin + changed_when: "admin.rc == 0" + failed_when: "(admin.rc > 0) and (admin.stderr is not defined or admin.stderr.find('already exists') == -1)" + +- name: Load tests + shell: "/var/lib/openqa/share/tests/fedora/templates | grep 'added => [1-9]'" + register: templates + changed_when: "templates.rc == 0" + failed_when: "1 != 1" diff --git a/roles/openqa/server/templates/client.conf.j2 b/roles/openqa/server/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/server/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/server/templates/database.ini.pgsql.j2 b/roles/openqa/server/templates/database.ini.pgsql.j2 new file mode 100644 index 0000000000..05712e2c80 --- /dev/null +++ b/roles/openqa/server/templates/database.ini.pgsql.j2 @@ -0,0 +1,9 @@ +[test] +dsn = dbi:SQLite:dbname=:memory: +on_connect_call = use_foreign_keys +on_connect_do = PRAGMA synchronous = OFF + +[production] +dsn = dbi:Pg:dbname={{ openqa_dbname }};host={{ openqa_dbhost }} +user = {{ openqa_dbuser }} +password = {{ openqa_dbpassword }} diff --git a/roles/openqa/server/templates/openqa.conf.httpd.j2 b/roles/openqa/server/templates/openqa.conf.httpd.j2 new file mode 100644 index 0000000000..dd8d9e6195 --- /dev/null +++ b/roles/openqa/server/templates/openqa.conf.httpd.j2 @@ -0,0 +1,4 @@ + + ServerName {{ external_hostname|default(ansible_nodename) }} + Include conf.d/openqa-common.inc + diff --git a/roles/openqa/worker/tasks/main.yml b/roles/openqa/worker/tasks/main.yml new file mode 100644 index 0000000000..70c4c70ed9 --- /dev/null +++ b/roles/openqa/worker/tasks/main.yml @@ -0,0 +1,45 @@ +# Required vars +# - openqa_workers +## integer - number of worker instances to create/run + +# Required vars with defaults +# - openqa_hostname +## string - hostname of openQA server to run jobs for +## default - localhost + +- name: Ensure DNF COPR plugin is available + dnf: pkg="dnf-command(copr)" state=present + tags: + - packages + +- name: Install openQA repo if needed + command: "dnf -y copr enable adamwill/openQA" + args: + creates: /etc/yum.repos.d/_copr_adamwill-openQA.repo + tags: + - config + +- name: Install packages + dnf: name={{ item }} state=present enablerepo=adamwill-openQA + with_items: + - openqa-worker + - libselinux-python + tags: + - packages + +- include: nfs-client.yml + when: openqa_hostname is defined and openqa_hostname != "localhost" + +- name: openQA client config + template: src=client.conf.j2 dest=/etc/openqa/client.conf owner=_openqa-worker group=root mode=0600 + tags: + - config + +- name: openQA worker config + template: src=workers.ini.j2 dest=/etc/openqa/workers.ini owner=_openqa-worker group=root mode=0644 + tags: + - config + +- name: Worker services + service: name=openqa-worker@{{ item }} enabled=yes state=started + with_sequence: "count={{ openqa_workers }}" diff --git a/roles/openqa/worker/tasks/nfs-client.yml b/roles/openqa/worker/tasks/nfs-client.yml new file mode 100644 index 0000000000..2eedd0ec3f --- /dev/null +++ b/roles/openqa/worker/tasks/nfs-client.yml @@ -0,0 +1,24 @@ +# Required vars +# - openqa_hostname +## string - hostname of the openQA server (we assume it is hosting the NFS mount) + +- name: Install NFS client + dnf: name=nfs-utils state=present + tags: + - packages + +# We don't check ownership as, after mounting, it's owned by whatever the +# UID of geekotest is on the server +- name: Ensure mount target exists + file: path=/var/lib/openqa/share state=directory mode=0755 + +- name: Create mount unit + template: src=var-lib-openqa-share.mount.j2 dest=/etc/systemd/system/var-lib-openqa-share.mount owner=root group=root mode=0644 + tags: + - config + +- name: Enable and start mount + service: name={{ item }} enabled=yes state=started + with_items: + - var-lib-openqa-share.mount + - remote-fs.target diff --git a/roles/openqa/worker/templates/client.conf.j2 b/roles/openqa/worker/templates/client.conf.j2 new file mode 100644 index 0000000000..609e86a2c5 --- /dev/null +++ b/roles/openqa/worker/templates/client.conf.j2 @@ -0,0 +1,3 @@ +[{{ openqa_hostname|default('localhost') }}] +key = {{ openqa_key }} +secret = {{ openqa_secret }} diff --git a/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 b/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 new file mode 100644 index 0000000000..3ec823f460 --- /dev/null +++ b/roles/openqa/worker/templates/var-lib-openqa-share.mount.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=openQA shared assets +ConditionPathExists=/var/lib/openqa/share + +[Mount] +What={{ openqa_hostname }}:/var/lib/openqa/share +Where=/var/lib/openqa/share +Type=nfs + +[Install] +WantedBy = remote-fs.target diff --git a/roles/openqa/worker/templates/workers.ini.j2 b/roles/openqa/worker/templates/workers.ini.j2 new file mode 100644 index 0000000000..b9665bc8fc --- /dev/null +++ b/roles/openqa/worker/templates/workers.ini.j2 @@ -0,0 +1,2 @@ +[global] +HOST = http://{{ openqa_hostname|default('localhost') }} diff --git a/roles/openqa/worker/vars/main.yml b/roles/openqa/worker/vars/main.yml new file mode 100644 index 0000000000..42aacce60c --- /dev/null +++ b/roles/openqa/worker/vars/main.yml @@ -0,0 +1 @@ +openqa_hostname: localhost diff --git a/roles/repo2json/files/rhel_to_json.py b/roles/repo2json/files/rhel_to_json.py index 63ba2520e3..2b500b4a5a 100644 --- a/roles/repo2json/files/rhel_to_json.py +++ b/roles/repo2json/files/rhel_to_json.py @@ -65,44 +65,10 @@ PATHS = { '/mnt/fedora/app/fi-repo/rhel/rhel7/', ], 'el6': [ - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-ha-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-ha-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-lb-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-lb-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-optional-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-optional-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-ha-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-ha-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-lb-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-lb-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-optional-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc64-server-optional-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-ha-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-ha-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-lb-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-lb-fastrack-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-optional-6', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-optional-fastrack-6', + '/mnt/fedora/app/fi-repo/rhel/rhel6/', ], 'el5': [ - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-fastrack-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-productivity-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-i386-server-vt-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc-server-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc-server-fastrack-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc-server-productivity-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-ppc-server-vt-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-fastrack-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-productivity-5/', - '/mnt/fedora/app/fi-repo/rhel/rhel-x86_64-server-vt-5/', + '/mnt/fedora/app/fi-repo/rhel/rhel5/', ], } diff --git a/roles/taskotron/taskotron-client/tasks/main.yml b/roles/taskotron/taskotron-client/tasks/main.yml index 60094559a2..d242342536 100644 --- a/roles/taskotron/taskotron-client/tasks/main.yml +++ b/roles/taskotron/taskotron-client/tasks/main.yml @@ -14,7 +14,6 @@ - PyYAML - libtaskotron - resultsdb_api - when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined - name: ensure packages required for taskotron tasks are installed (yum) yum: name={{ item }} state=latest enablerepo={{ extra_enablerepos }}