taskotron dev: allow to use multiple slaves per host

inventory/group_vars/taskotron-dev

@@ -68,6 +68,5 @@ buildmaster_home: /srv/buildmaster
 buildmaster_template: taskotron.master.cfg.j2
 buildmaster_user: buildmaster
 
-buildslave_dir: /home/buildslave/slave
 buildslave_port: 9989
 buildslave_ssh_pubkey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDqNpJlS14KKQQ8gAaXo0untJRccLJSrdUt/hWlpZ9Oad5KKYoRNznsGK3j9WCO4YDFXKGGxBrpqdyegPLO4J9zHmtlUPNfM4kbXVlVtIMGGKmBi9rQPNRpeclUpsjlo3uF9IqIIUSS1IMcx1X1oz60kmrAWO9+Z7mYs/AuCCeomlET3OU7paQYxL5Dgs2n6/IjhvoCitN9T9bBf91nLuRh+EtEbBL5ztZ1kVJxj5oTQN7iGKw2V0tW0MJ1WtnK2/j4v9MivnZPSPPKDysveNze2Vy9kioQACvrvjyHMIQrc+2YS/38D/fv7e4WhhuoOGT5Tip20PPspiCQRiSUBDN1 buildslave@taskotron-stg.fedoraproject.org'

inventory/group_vars/taskotron-dev-clients

@@ -21,10 +21,7 @@ extra_enablerepos: ''
 # buildslave config
 ############################################################
 
-slave_user: buildslave
+slaves_group: buildslaves
-slave_home: /home/buildslave/
-slave_dir: /home/buildslave/slave
-slave_user: buildslave
 buildmaster: 10.5.124.181
 buildslave_port: 9989
 taskotron_admin_email: taskotron-admin-members@fedoraproject.org

inventory/host_vars/taskotron-client26.qa.fedoraproject.org

@@ -10,5 +10,7 @@ vmhost: qa10.qa.fedoraproject.org
 datacenter: phx2
 
 short_hostname: taskotron-client26.qa
-buildslave_name: taskotron-client26
+slaves:
+  - { user: taskotron-client26-1, home: /home/taskotron-client26-1, dir: /home/taskotron-client26-1/slave }
+  - { user: taskotron-client26-2, home: /home/taskotron-client26-2, dir: /home/taskotron-client26-2/slave }
 fas_client_groups: sysadmin-qa,sysadmin-main

inventory/host_vars/taskotron-client27.qa.fedoraproject.org

@@ -12,5 +12,7 @@ datacenter: phx2
 #libdir: /usr/lib
 
 short_hostname: taskotron-client27.qa
-buildslave_name: taskotron-client27
+slaves:
+  - { user: taskotron-client27-1, home: /home/taskotron-client27-1, dir: /home/taskotron-client27-1/slave }
+  - { user: taskotron-client27-2, home: /home/taskotron-client27-2, dir: /home/taskotron-client27-2/slave }
 fas_client_groups: sysadmin-qa,sysadmin-main

inventory/host_vars/taskotron-client28.qa.fedoraproject.org

@@ -10,5 +10,7 @@ vmhost: qa10.qa.fedoraproject.org
 datacenter: phx2
 
 short_hostname: taskotron-client28.qa
-buildslave_name: taskotron-client28
+slaves:
+  - { user: taskotron-client28-1, home: /home/taskotron-client28-1, dir: /home/taskotron-client28-1/slave }
+  - { user: taskotron-client28-2, home: /home/taskotron-client28-2, dir: /home/taskotron-client28-2/slave }
 fas_client_groups: sysadmin-qa,sysadmin-main

inventory/host_vars/taskotron-client29.qa.fedoraproject.org

@@ -11,5 +11,7 @@ datacenter: phx2
 libdir: /usr/lib
 
 short_hostname: taskotron-client29.qa
-buildslave_name: taskotron-client29
+slaves:
+  - { user: taskotron-client29-1, home: /home/taskotron-client29-1, dir: /home/taskotron-client29-1/slave }
+  - { user: taskotron-client29-2, home: /home/taskotron-client29-2, dir: /home/taskotron-client29-2/slave }
 fas_client_groups: sysadmin-qa,sysadmin-main

inventory/host_vars/taskotron-dev01.qa.fedoraproject.org

@@ -47,13 +47,21 @@ public_hostname: taskotron-dev.fedoraproject.org
 buildmaster: 10.5.124.181
 
 buildslaves:
-    - taskotron-client26
+    - taskotron-client26-1
-    - taskotron-client27
+    - taskotron-client26-2
-    - taskotron-client28
+    - taskotron-client27-1
-    - taskotron-client29
+    - taskotron-client27-2
+    - taskotron-client28-1
+    - taskotron-client28-2
+    - taskotron-client29-1
+    - taskotron-client29-2
 i386_buildslaves:
-    - taskotron-client29
+    - taskotron-client29-1
+    - taskotron-client29-2
 x86_64_buildslaves:
-    - taskotron-client27
+    - taskotron-client26-1
-    - taskotron-client26
+    - taskotron-client26-2
-    - taskotron-client28
+    - taskotron-client27-1
+    - taskotron-client27-2
+    - taskotron-client28-1
+    - taskotron-client28-2

roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2

@@ -151,11 +151,18 @@ from buildbot.steps.master import MasterShellCommand
 
 factory = BuildFactory()
 
+{% if deployment_type in ['prod', 'stg', 'local'] %}
 # clean out /var/tmp/taskotron (see T253)
 factory.addStep(ShellCommand(command="rm -rf /var/tmp/taskotron/*", name="rm_tmp", descriptionDone=['Clean tmp']))
 
 # clean the log (see T230)
 factory.addStep(ShellCommand(command=["rm", "-f", "/var/log/taskotron/taskotron.log"], name="rm_log", descriptionDone=['Clean log']))
+{% endif %}
+
+{% if deployment_type == 'dev' %}
+# clean out /var/tmp/taskotron (see T253)
+factory.addStep(ShellCommand(command=["rm", "-rf", Interpolate("/var/tmp/taskotron/%(prop:slavename)s/*")], name="rm_tmp", descriptionDone=['Clean tmp']))
+{% endif %}
 
 # check out the source
 factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%(prop:taskname)s/'),
@@ -172,8 +179,12 @@ factory.addStep(ShellCommand(command=["runtask",
                              descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)s')],
                              name='runtask',
                              timeout=2400,
+{% if deployment_type == 'dev' %}
+                             logfiles={'taskotron.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron.log'), }}))
+{% endif %}
+{% if deployment_type in ['prod', 'stg', 'local'] %}
                              logfiles={'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log', }}))
-
+{% endif %}
 
 # create artifacts dir on master
 factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s')],
@@ -183,10 +194,14 @@ factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('
 factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'),
                                 masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output')))
 
+
+{% if deployment_type in ['prod', 'stg', 'local'] %}
 # copy taskotron log to master
 factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log',
                            masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'),
                            mode=0644))
+{% endif %}
+
 import datetime
 from buildbot.process.properties import renderer
 

roles/taskotron/buildslave-configure/tasks/main.yml

@@ -2,11 +2,13 @@
   sudo: true
   sudo_user: "{{ slave_user }}"
   command: creates={{ slave_dir }} buildslave create-slave {{ slave_dir }} {{ buildmaster }} buildslave passwd
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
 
 - name: generate slave config
   sudo: true
   sudo_user: "{{ slave_user }}"
   template: src=buildbot.tac.j2 dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
 
 - name: generate slave info
   sudo: true
@@ -15,26 +17,95 @@
   with_items:
     - admin
     - host
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
 
 - name: create slave ssh directory
-  when: buildslave_public_sshkey_file is defined
+  when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
   file: path={{ slave_home }}/.ssh owner={{ slave_user }} group={{ slave_user }} mode=0700 state=directory
 
 - name: install slave ssh private key
-  when: buildslave_private_sshkey_file is defined
+  when: buildslave_private_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
   copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa owner={{ slave_user }} group={{ slave_user }} mode=0600
 
 - name: install slave ssh public key
-  when: buildslave_public_sshkey_file is defined
+  when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
   copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa.pub owner={{ slave_user }} group={{ slave_user }} mode=0644
 
 - name: make sure master is in known_hosts
-  when: buildslave_public_sshkey_file is defined
+  when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
   lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }}
 
 - name: generate buildslave service file
   template: src=buildslave.service.j2 dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
 
 - name: start and enable buildslave service
   service: name=buildslave enabled=yes state=started
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
 
+
+- name: create slave
+  sudo: true
+  sudo_user: '{{ item.user }}'
+  command: creates={{ item.dir }} buildslave create-slave {{ item.dir }} {{ buildmaster }} buildslave passwd
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type == 'dev'
+
+- name: generate slave config
+  sudo: true
+  sudo_user: '{{ item.user }}'
+  template: src=buildbot.tac.j2 dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type == 'dev'
+
+- name: generate slave admin info
+  sudo: true
+  sudo_user: '{{ item.user }}'
+  template: src=admin.j2 dest={{ item.dir }}/info/admin mode=0644 owner={{ item.user }} group={{ slaves_group }}
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type == 'dev'
+
+- name: generate slave host info
+  sudo: true
+  sudo_user: '{{ item.user }}'
+  template: src=host.j2 dest={{ item.dir }}/info/host mode=0644 owner={{ item.user }} group={{ slaves_group }}
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type == 'dev'
+
+- name: create slave ssh directory
+  when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
+  file: path={{ item.home }}/.ssh owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
+  with_items:
+    - '{{ slaves }}'
+
+- name: install slave ssh private key
+  when: buildslave_private_sshkey_file is defined and deployment_type == 'dev'
+  copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa owner={{ item.user }} group={{ slaves_group }} mode=0600
+  with_items:
+    - '{{ slaves }}'
+
+- name: install slave ssh public key
+  when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
+  copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa.pub owner={{ item.user }} group={{ slaves_group }} mode=0644
+  with_items:
+    - '{{ slaves }}'
+
+- name: make sure master is in known_hosts
+  when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
+  lineinfile: dest={{ item.home }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ item.user }} group={{ slaves_group }}
+  with_items:
+    - '{{ slaves }}'
+
+- name: generate buildslave service file
+  template: src=buildslave@.service.j2 dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0744
+  when: deployment_type == 'dev'
+
+- name: start and enable buildslave services
+  service: name=buildslave@{{ item.user }} enabled=yes state=started
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type == 'dev'

roles/taskotron/buildslave-configure/templates/buildbot.tac.j2

@@ -4,7 +4,12 @@ import os
 from twisted.application import service
 from buildslave.bot import BuildSlave
 
+{% if deployment_type in ['prod', 'stg', 'local'] %}
 basedir = r'/home/buildslave/slave'
+{% endif %}
+{% if deployment_type == 'dev' %}
+basedir = r'{{ item.dir }}'
+{% endif %}
 rotateLength = 10000000
 maxRotatedFiles = 10
 
@@ -29,7 +34,12 @@ except ImportError:
 
 buildmaster_host = '{{ buildmaster }}'
 port = 9989
+{% if deployment_type in ['prod', 'stg', 'local'] %}
 slavename = '{{ buildslave_name }}'
+{% endif %}
+{% if deployment_type == 'dev' %}
+slavename = '{{ item.user }}'
+{% endif %}
 {% if deployment_type == 'dev' %}
 passwd = '{{ dev_buildslave_password }}'
 {% endif %}

roles/taskotron/buildslave-configure/templates/buildslave@.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Buildslave for taskotron
+After=network.target
+
+[Service]
+Type=forking
+PIDFile=/home/%i/slave/twistd.pid
+ExecStart=/bin/buildslave start /home/%i/slave/
+ExecStop=/bin/buildslave stop /home/%i/slave/
+User=%i
+Group={{ slaves_group }}
+
+[Install]
+WantedBy=multi-user.target

roles/taskotron/buildslave/tasks/main.yml

@@ -35,3 +35,13 @@
 
 - name: add the buildslave user
   user: name=buildslave
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
+
+- name: add buildslaves group
+  group: name={{ slaves_group }}
+  when: deployment_type == 'dev'
+
+- name: add buildslave users
+  user: name={{ item.user }} group={{ slaves_group }} home={{ item.home }}
+  with_items: slaves
+  when: deployment_type == 'dev'

roles/taskotron/taskotron-client/tasks/main.yml

@@ -44,3 +44,8 @@
 
 - name: create /var/log/taskotron for task logfiles
   file: path=/var/log/taskotron state=directory owner={{ slave_user }} group={{ slave_user }} mode=1755
+  when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
+
+- name: create /var/log/taskotron for task logfiles
+  file: path=/var/log/taskotron state=directory owner=root group={{ slaves_group }} mode=1775
+  when: deployment_type == 'dev'

roles/taskotron/taskotron-client/templates/taskotron.yaml.j2

@@ -1,28 +1,51 @@
 ## Main configuration file for Taskotron
 ## The file is in YAML syntax, read more about it at:
 ## http://en.wikipedia.org/wiki/Yaml
-## FIXME: add a link to Taskotron docs
+## libtaskotron docs live at:
+## https://docs.qadevel.cloud.fedoraproject.org/libtaskotron/latest/
+
 
 ## ==== GENERAL section ====
+## Config profile selection.
 ## There are two major config profiles in Taskotron - development and
 ## production.
-## /Development/ profile is used for developing libtaskotron, developing checks
+## Development profile is used for developing libtaskotron, developing checks
 ## based on libtaskotron and local execution of these checks.
-## /Production/ profile is used for deploying Taskotron as a service on a
+## Production profile is used for deploying Taskotron as a service on a
 ## server, periodically executing the checks and reporting results to relevant
 ## result consumers.
-## *The default profile is /development/*. If you want to switch to the
+## The default profile is 'development'. If you want to switch to the
-## /production/ profile, uncomment the following line.
+## 'production' profile, uncomment the following line.
 ## You can also switch a profile temporarily by using TASKOTRON_PROFILE=name
 ## environment variable, it has a higher priority. All other options set in this
 ## file still apply of course.
 ## [choices: production, development; default: development]
 profile: production
 
+## Task execution mode. The tasks can be executed locally (on the current
+## machine) or remotely in a disposable virtual machine. Local execution is
+## dangerous with destructive or untrusted tasks, remote execution requires
+## some additional setup (see Taskotron documentation).
+## Remote execution is done through libvirt, it creates a throwaway virtual
+## machine from a specified disk image, executes the task and deletes the
+## machine.
+## Local execution is the default mode for development profile and remote
+## execution for production profile.
+## [choices: local, libvirt; default: local for development, libvirt for production]
+runtask_mode: local
+
+
+## ==== SCHEDULING section ====
+## This section holds options related to the scheduling and execution system,
+## currently we use Buildbot
+
+## name of step in buildbot that executes the task
+buildbot_task_step = 'runtask'
+
 
 ## ==== REPORTING section ====
 ## This section controls which result reports you want to send after the test
-## execution is complete
+## execution is complete.
 
 ## Overall setting whether to send any reports at all. If this is False, no
 ## reports will be sent anywhere, regardless of your configuration for
@@ -32,18 +55,23 @@ reporting_enabled: True
 
 ## If True, test results (for selected tests) will be sent as comments to
 ## Fedora Update System (Bodhi). This requires that you have Bodhi
-## credentials filled in secrets.conf.
+## credentials filled in secrets.conf. See also 'reporting_enabled' option.
 report_to_bodhi: {{ client_report_to_bodhi }}
 
-## Whether to send test results to the configured ResultsDB server.
+## Whether to send test results to the configured ResultsDB server. See also
+## 'reporting_enabled' option.
 report_to_resultsdb: True
 
 
 ## ==== RESOURCES section ====
-## This section specifies access details to various external services
+## This section specifies access details to various external services.
+##
+## Note: Try to keep custom URL addresses without a trailing slash. Otherwise
+## the rendered URLs might end up containing double slashes, which some
+## application servers don't handle gracefully (e.g. Flask's internal app
+## server werkzeug).
 
 ## URL of Koji instance used for querying about new builds
-#koji_url: http://koji.fedoraproject.org/kojihub
 koji_url: {{ kojihub_url }}
 
 ## URL of repository of all the RPM packages built in Koji
@@ -52,25 +80,36 @@ koji_url: {{ kojihub_url }}
 ## URL of Bodhi instance used for communication about package updates
 bodhi_server: {{ bodhi_server }}
 
-## URL of ResultsDB server API interface, which can store all test results
+## Whether to use staging Bodhi instance instead of production (the
+## default one).
+#bodhi_staging: False
+
+## URL of ResultsDB server API interface, which can store all test results.
+## Please make sure the URL doesn't have a trailing slash.
 resultsdb_server: {{ resultsdb_server }}
 
 ## URL of ResultsDB frontend, which displays results from ResultsDB.
 ## Please make sure the URL doesn't have a trailing slash.
 resultsdb_frontend: {{ resultsdb_external_url }}
 
-## URL of ExecDB server API interface, which tracks task execution status
+## URL of ExecDB server API interface, which tracks task execution status.
+## Please make sure the URL doesn't have a trailing slash.
 execdb_server: {{ execdb_external_url }}
 
-## URL of taskotron buildmaster
+## URL of taskotron buildmaster, to construct log URLs from.
+## Please make sure the URL doesn't have a trailing slash.
 taskotron_master: {{ taskotron_master }}
 
 ## URL of artifacts base directory, to construct artifacts URLs from.
 ## Please make sure the URL doesn't have a trailing slash.
 artifacts_baseurl: {{ artifacts_base_url }}
 
-## name of step in buildbot that executes the task
+## Whether to cache downloaded files to speed up subsequent downloads. If True,
-buildbot_task_step: 'runtask'
+## files will be downloaded to a common directory specified by "cachedir". At
+## the moment, Taskotron only supports Koji RPM downloads to be cached.
+## [default: False for production, True for development]
+#download_cache_enabled: False
+
 
 ## ==== BODHI EMAIL section ====
 ## These configuration options affect how Taskotron decideds to send emails
@@ -79,14 +118,42 @@ buildbot_task_step: 'runtask'
 ## How long (in minutes) should we wait before allowing consequent test to
 ## re-post a 'FAILED' comment into Bodhi once again.
 ## By default 3 days (3*24*60 = 4320).
-#bodhi_email_failed_span: 4320
+#bodhi_posting_comments_span: 4320
 
 
 ## ==== PATHS section ====
 ## Location of various pieces of the project.
 
-## The main log file location for Taskotron
+## The location of log files for Taskotron
-#logfile: /var/log/taskotron/taskotron.log
+#logdir: /var/log/taskotron
+
+## The location of task files (git checkout) when running in disposable clients mode
+#client_taskdir: /var/tmp/taskotron/taskdir
+
+## The location of temporary files for Taskotron
+#tmpdir: /var/tmp/taskotron
+
+## The location of artifacts produced by checks
+#artifactsdir: /var/lib/taskotron/artifacts
+
+## The location of cached files downloaded by Taskotron
+#cachedir: /var/cache/taskotron
+
+
+## ==== LOGGING section ====
+## This section contains configuration of logging.
+
+## Configuration of logging level. Here can be configured which messages
+## will be logged. You can specify different level for logging to standard
+## output (option log_level_stream) and logging to file (log_level_file).
+## Possible values can be found here:
+## https://docs.python.org/2.7/library/logging.html#logging-levels
+#log_level_stream: INFO
+#log_level_file: DEBUG
+
+## If True, logging to file will be enabled.
+## [default: True for production, False for development]
+#log_file_enabled: True
 
 
 ## ==== SECRETS section ====
@@ -98,3 +165,9 @@ buildbot_task_step: 'runtask'
 fas_username: '{{ taskotron_fas_user }}'
 fas_password: '{{ taskotron_fas_password}}'
 
+## SSH private key location. Used for remote task execution, when connecting to
+## VMs and remote machines. If your systems are not configured for automatic
+## connection (private keys located in SSH's standard search path), specify a
+## path to the private key here. An empty value (the default) means to rely on
+## the standard search path only.
+#ssh_privkey: /path/to/private.key