taskotron dev: allow to use multiple slaves per host

This commit is contained in:
Martin Krizek 2015-11-02 20:34:03 +00:00
parent daee0aa950
commit 808742f874
14 changed files with 250 additions and 40 deletions

View file

@ -68,6 +68,5 @@ buildmaster_home: /srv/buildmaster
buildmaster_template: taskotron.master.cfg.j2
buildmaster_user: buildmaster
buildslave_dir: /home/buildslave/slave
buildslave_port: 9989
buildslave_ssh_pubkey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDqNpJlS14KKQQ8gAaXo0untJRccLJSrdUt/hWlpZ9Oad5KKYoRNznsGK3j9WCO4YDFXKGGxBrpqdyegPLO4J9zHmtlUPNfM4kbXVlVtIMGGKmBi9rQPNRpeclUpsjlo3uF9IqIIUSS1IMcx1X1oz60kmrAWO9+Z7mYs/AuCCeomlET3OU7paQYxL5Dgs2n6/IjhvoCitN9T9bBf91nLuRh+EtEbBL5ztZ1kVJxj5oTQN7iGKw2V0tW0MJ1WtnK2/j4v9MivnZPSPPKDysveNze2Vy9kioQACvrvjyHMIQrc+2YS/38D/fv7e4WhhuoOGT5Tip20PPspiCQRiSUBDN1 buildslave@taskotron-stg.fedoraproject.org'

View file

@ -21,10 +21,7 @@ extra_enablerepos: ''
# buildslave config
############################################################
slave_user: buildslave
slave_home: /home/buildslave/
slave_dir: /home/buildslave/slave
slave_user: buildslave
slaves_group: buildslaves
buildmaster: 10.5.124.181
buildslave_port: 9989
taskotron_admin_email: taskotron-admin-members@fedoraproject.org

View file

@ -10,5 +10,7 @@ vmhost: qa10.qa.fedoraproject.org
datacenter: phx2
short_hostname: taskotron-client26.qa
buildslave_name: taskotron-client26
slaves:
- { user: taskotron-client26-1, home: /home/taskotron-client26-1, dir: /home/taskotron-client26-1/slave }
- { user: taskotron-client26-2, home: /home/taskotron-client26-2, dir: /home/taskotron-client26-2/slave }
fas_client_groups: sysadmin-qa,sysadmin-main

View file

@ -12,5 +12,7 @@ datacenter: phx2
#libdir: /usr/lib
short_hostname: taskotron-client27.qa
buildslave_name: taskotron-client27
slaves:
- { user: taskotron-client27-1, home: /home/taskotron-client27-1, dir: /home/taskotron-client27-1/slave }
- { user: taskotron-client27-2, home: /home/taskotron-client27-2, dir: /home/taskotron-client27-2/slave }
fas_client_groups: sysadmin-qa,sysadmin-main

View file

@ -10,5 +10,7 @@ vmhost: qa10.qa.fedoraproject.org
datacenter: phx2
short_hostname: taskotron-client28.qa
buildslave_name: taskotron-client28
slaves:
- { user: taskotron-client28-1, home: /home/taskotron-client28-1, dir: /home/taskotron-client28-1/slave }
- { user: taskotron-client28-2, home: /home/taskotron-client28-2, dir: /home/taskotron-client28-2/slave }
fas_client_groups: sysadmin-qa,sysadmin-main

View file

@ -11,5 +11,7 @@ datacenter: phx2
libdir: /usr/lib
short_hostname: taskotron-client29.qa
buildslave_name: taskotron-client29
slaves:
- { user: taskotron-client29-1, home: /home/taskotron-client29-1, dir: /home/taskotron-client29-1/slave }
- { user: taskotron-client29-2, home: /home/taskotron-client29-2, dir: /home/taskotron-client29-2/slave }
fas_client_groups: sysadmin-qa,sysadmin-main

View file

@ -47,13 +47,21 @@ public_hostname: taskotron-dev.fedoraproject.org
buildmaster: 10.5.124.181
buildslaves:
- taskotron-client26
- taskotron-client27
- taskotron-client28
- taskotron-client29
- taskotron-client26-1
- taskotron-client26-2
- taskotron-client27-1
- taskotron-client27-2
- taskotron-client28-1
- taskotron-client28-2
- taskotron-client29-1
- taskotron-client29-2
i386_buildslaves:
- taskotron-client29
- taskotron-client29-1
- taskotron-client29-2
x86_64_buildslaves:
- taskotron-client27
- taskotron-client26
- taskotron-client28
- taskotron-client26-1
- taskotron-client26-2
- taskotron-client27-1
- taskotron-client27-2
- taskotron-client28-1
- taskotron-client28-2

View file

@ -151,11 +151,18 @@ from buildbot.steps.master import MasterShellCommand
factory = BuildFactory()
{% if deployment_type in ['prod', 'stg', 'local'] %}
# clean out /var/tmp/taskotron (see T253)
factory.addStep(ShellCommand(command="rm -rf /var/tmp/taskotron/*", name="rm_tmp", descriptionDone=['Clean tmp']))
# clean the log (see T230)
factory.addStep(ShellCommand(command=["rm", "-f", "/var/log/taskotron/taskotron.log"], name="rm_log", descriptionDone=['Clean log']))
{% endif %}
{% if deployment_type == 'dev' %}
# clean out /var/tmp/taskotron (see T253)
factory.addStep(ShellCommand(command=["rm", "-rf", Interpolate("/var/tmp/taskotron/%(prop:slavename)s/*")], name="rm_tmp", descriptionDone=['Clean tmp']))
{% endif %}
# check out the source
factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%(prop:taskname)s/'),
@ -172,8 +179,12 @@ factory.addStep(ShellCommand(command=["runtask",
descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)s')],
name='runtask',
timeout=2400,
{% if deployment_type == 'dev' %}
logfiles={'taskotron.log': {'filename': Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron.log'), }}))
{% endif %}
{% if deployment_type in ['prod', 'stg', 'local'] %}
logfiles={'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log', }}))
{% endif %}
# create artifacts dir on master
factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s')],
@ -183,10 +194,14 @@ factory.addStep(MasterShellCommand(command=["mkdir", '-m', '0755', Interpolate('
factory.addStep(DirectoryUpload(slavesrc=Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'),
masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/task_output')))
{% if deployment_type in ['prod', 'stg', 'local'] %}
# copy taskotron log to master
factory.addStep(FileUpload(slavesrc='/var/log/taskotron/taskotron.log',
masterdest=Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'),
mode=0644))
{% endif %}
import datetime
from buildbot.process.properties import renderer

View file

@ -2,11 +2,13 @@
sudo: true
sudo_user: "{{ slave_user }}"
command: creates={{ slave_dir }} buildslave create-slave {{ slave_dir }} {{ buildmaster }} buildslave passwd
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: generate slave config
sudo: true
sudo_user: "{{ slave_user }}"
template: src=buildbot.tac.j2 dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: generate slave info
sudo: true
@ -15,26 +17,95 @@
with_items:
- admin
- host
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: create slave ssh directory
when: buildslave_public_sshkey_file is defined
when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
file: path={{ slave_home }}/.ssh owner={{ slave_user }} group={{ slave_user }} mode=0700 state=directory
- name: install slave ssh private key
when: buildslave_private_sshkey_file is defined
when: buildslave_private_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa owner={{ slave_user }} group={{ slave_user }} mode=0600
- name: install slave ssh public key
when: buildslave_public_sshkey_file is defined
when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa.pub owner={{ slave_user }} group={{ slave_user }} mode=0644
- name: make sure master is in known_hosts
when: buildslave_public_sshkey_file is defined
when: buildslave_public_sshkey_file is defined and (deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local')
lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }}
- name: generate buildslave service file
template: src=buildslave.service.j2 dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: start and enable buildslave service
service: name=buildslave enabled=yes state=started
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: create slave
sudo: true
sudo_user: '{{ item.user }}'
command: creates={{ item.dir }} buildslave create-slave {{ item.dir }} {{ buildmaster }} buildslave passwd
with_items:
- '{{ slaves }}'
when: deployment_type == 'dev'
- name: generate slave config
sudo: true
sudo_user: '{{ item.user }}'
template: src=buildbot.tac.j2 dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves }}'
when: deployment_type == 'dev'
- name: generate slave admin info
sudo: true
sudo_user: '{{ item.user }}'
template: src=admin.j2 dest={{ item.dir }}/info/admin mode=0644 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves }}'
when: deployment_type == 'dev'
- name: generate slave host info
sudo: true
sudo_user: '{{ item.user }}'
template: src=host.j2 dest={{ item.dir }}/info/host mode=0644 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves }}'
when: deployment_type == 'dev'
- name: create slave ssh directory
when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
file: path={{ item.home }}/.ssh owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
with_items:
- '{{ slaves }}'
- name: install slave ssh private key
when: buildslave_private_sshkey_file is defined and deployment_type == 'dev'
copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa owner={{ item.user }} group={{ slaves_group }} mode=0600
with_items:
- '{{ slaves }}'
- name: install slave ssh public key
when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa.pub owner={{ item.user }} group={{ slaves_group }} mode=0644
with_items:
- '{{ slaves }}'
- name: make sure master is in known_hosts
when: buildslave_public_sshkey_file is defined and deployment_type == 'dev'
lineinfile: dest={{ item.home }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves }}'
- name: generate buildslave service file
template: src=buildslave@.service.j2 dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0744
when: deployment_type == 'dev'
- name: start and enable buildslave services
service: name=buildslave@{{ item.user }} enabled=yes state=started
with_items:
- '{{ slaves }}'
when: deployment_type == 'dev'

View file

@ -4,7 +4,12 @@ import os
from twisted.application import service
from buildslave.bot import BuildSlave
{% if deployment_type in ['prod', 'stg', 'local'] %}
basedir = r'/home/buildslave/slave'
{% endif %}
{% if deployment_type == 'dev' %}
basedir = r'{{ item.dir }}'
{% endif %}
rotateLength = 10000000
maxRotatedFiles = 10
@ -29,7 +34,12 @@ except ImportError:
buildmaster_host = '{{ buildmaster }}'
port = 9989
{% if deployment_type in ['prod', 'stg', 'local'] %}
slavename = '{{ buildslave_name }}'
{% endif %}
{% if deployment_type == 'dev' %}
slavename = '{{ item.user }}'
{% endif %}
{% if deployment_type == 'dev' %}
passwd = '{{ dev_buildslave_password }}'
{% endif %}

View file

@ -0,0 +1,14 @@
[Unit]
Description=Buildslave for taskotron
After=network.target
[Service]
Type=forking
PIDFile=/home/%i/slave/twistd.pid
ExecStart=/bin/buildslave start /home/%i/slave/
ExecStop=/bin/buildslave stop /home/%i/slave/
User=%i
Group={{ slaves_group }}
[Install]
WantedBy=multi-user.target

View file

@ -35,3 +35,13 @@
- name: add the buildslave user
user: name=buildslave
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: add buildslaves group
group: name={{ slaves_group }}
when: deployment_type == 'dev'
- name: add buildslave users
user: name={{ item.user }} group={{ slaves_group }} home={{ item.home }}
with_items: slaves
when: deployment_type == 'dev'

View file

@ -44,3 +44,8 @@
- name: create /var/log/taskotron for task logfiles
file: path=/var/log/taskotron state=directory owner={{ slave_user }} group={{ slave_user }} mode=1755
when: deployment_type == 'prod' or deployment_type == 'stg' or deployment_type == 'local'
- name: create /var/log/taskotron for task logfiles
file: path=/var/log/taskotron state=directory owner=root group={{ slaves_group }} mode=1775
when: deployment_type == 'dev'

View file

@ -1,28 +1,51 @@
## Main configuration file for Taskotron
## The file is in YAML syntax, read more about it at:
## http://en.wikipedia.org/wiki/Yaml
## FIXME: add a link to Taskotron docs
## libtaskotron docs live at:
## https://docs.qadevel.cloud.fedoraproject.org/libtaskotron/latest/
## ==== GENERAL section ====
## Config profile selection.
## There are two major config profiles in Taskotron - development and
## production.
## /Development/ profile is used for developing libtaskotron, developing checks
## Development profile is used for developing libtaskotron, developing checks
## based on libtaskotron and local execution of these checks.
## /Production/ profile is used for deploying Taskotron as a service on a
## Production profile is used for deploying Taskotron as a service on a
## server, periodically executing the checks and reporting results to relevant
## result consumers.
## *The default profile is /development/*. If you want to switch to the
## /production/ profile, uncomment the following line.
## The default profile is 'development'. If you want to switch to the
## 'production' profile, uncomment the following line.
## You can also switch a profile temporarily by using TASKOTRON_PROFILE=name
## environment variable, it has a higher priority. All other options set in this
## file still apply of course.
## [choices: production, development; default: development]
profile: production
## Task execution mode. The tasks can be executed locally (on the current
## machine) or remotely in a disposable virtual machine. Local execution is
## dangerous with destructive or untrusted tasks, remote execution requires
## some additional setup (see Taskotron documentation).
## Remote execution is done through libvirt, it creates a throwaway virtual
## machine from a specified disk image, executes the task and deletes the
## machine.
## Local execution is the default mode for development profile and remote
## execution for production profile.
## [choices: local, libvirt; default: local for development, libvirt for production]
runtask_mode: local
## ==== SCHEDULING section ====
## This section holds options related to the scheduling and execution system,
## currently we use Buildbot
## name of step in buildbot that executes the task
buildbot_task_step = 'runtask'
## ==== REPORTING section ====
## This section controls which result reports you want to send after the test
## execution is complete
## execution is complete.
## Overall setting whether to send any reports at all. If this is False, no
## reports will be sent anywhere, regardless of your configuration for
@ -32,18 +55,23 @@ reporting_enabled: True
## If True, test results (for selected tests) will be sent as comments to
## Fedora Update System (Bodhi). This requires that you have Bodhi
## credentials filled in secrets.conf.
## credentials filled in secrets.conf. See also 'reporting_enabled' option.
report_to_bodhi: {{ client_report_to_bodhi }}
## Whether to send test results to the configured ResultsDB server.
## Whether to send test results to the configured ResultsDB server. See also
## 'reporting_enabled' option.
report_to_resultsdb: True
## ==== RESOURCES section ====
## This section specifies access details to various external services
## This section specifies access details to various external services.
##
## Note: Try to keep custom URL addresses without a trailing slash. Otherwise
## the rendered URLs might end up containing double slashes, which some
## application servers don't handle gracefully (e.g. Flask's internal app
## server werkzeug).
## URL of Koji instance used for querying about new builds
#koji_url: http://koji.fedoraproject.org/kojihub
koji_url: {{ kojihub_url }}
## URL of repository of all the RPM packages built in Koji
@ -52,25 +80,36 @@ koji_url: {{ kojihub_url }}
## URL of Bodhi instance used for communication about package updates
bodhi_server: {{ bodhi_server }}
## URL of ResultsDB server API interface, which can store all test results
## Whether to use staging Bodhi instance instead of production (the
## default one).
#bodhi_staging: False
## URL of ResultsDB server API interface, which can store all test results.
## Please make sure the URL doesn't have a trailing slash.
resultsdb_server: {{ resultsdb_server }}
## URL of ResultsDB frontend, which displays results from ResultsDB.
## Please make sure the URL doesn't have a trailing slash.
resultsdb_frontend: {{ resultsdb_external_url }}
## URL of ExecDB server API interface, which tracks task execution status
## URL of ExecDB server API interface, which tracks task execution status.
## Please make sure the URL doesn't have a trailing slash.
execdb_server: {{ execdb_external_url }}
## URL of taskotron buildmaster
## URL of taskotron buildmaster, to construct log URLs from.
## Please make sure the URL doesn't have a trailing slash.
taskotron_master: {{ taskotron_master }}
## URL of artifacts base directory, to construct artifacts URLs from.
## Please make sure the URL doesn't have a trailing slash.
artifacts_baseurl: {{ artifacts_base_url }}
## name of step in buildbot that executes the task
buildbot_task_step: 'runtask'
## Whether to cache downloaded files to speed up subsequent downloads. If True,
## files will be downloaded to a common directory specified by "cachedir". At
## the moment, Taskotron only supports Koji RPM downloads to be cached.
## [default: False for production, True for development]
#download_cache_enabled: False
## ==== BODHI EMAIL section ====
## These configuration options affect how Taskotron decideds to send emails
@ -79,14 +118,42 @@ buildbot_task_step: 'runtask'
## How long (in minutes) should we wait before allowing consequent test to
## re-post a 'FAILED' comment into Bodhi once again.
## By default 3 days (3*24*60 = 4320).
#bodhi_email_failed_span: 4320
#bodhi_posting_comments_span: 4320
## ==== PATHS section ====
## Location of various pieces of the project.
## The main log file location for Taskotron
#logfile: /var/log/taskotron/taskotron.log
## The location of log files for Taskotron
#logdir: /var/log/taskotron
## The location of task files (git checkout) when running in disposable clients mode
#client_taskdir: /var/tmp/taskotron/taskdir
## The location of temporary files for Taskotron
#tmpdir: /var/tmp/taskotron
## The location of artifacts produced by checks
#artifactsdir: /var/lib/taskotron/artifacts
## The location of cached files downloaded by Taskotron
#cachedir: /var/cache/taskotron
## ==== LOGGING section ====
## This section contains configuration of logging.
## Configuration of logging level. Here can be configured which messages
## will be logged. You can specify different level for logging to standard
## output (option log_level_stream) and logging to file (log_level_file).
## Possible values can be found here:
## https://docs.python.org/2.7/library/logging.html#logging-levels
#log_level_stream: INFO
#log_level_file: DEBUG
## If True, logging to file will be enabled.
## [default: True for production, False for development]
#log_file_enabled: True
## ==== SECRETS section ====
@ -98,3 +165,9 @@ buildbot_task_step: 'runtask'
fas_username: '{{ taskotron_fas_user }}'
fas_password: '{{ taskotron_fas_password}}'
## SSH private key location. Used for remote task execution, when connecting to
## VMs and remote machines. If your systems are not configured for automatic
## connection (private keys located in SSH's standard search path), specify a
## path to the private key here. An empty value (the default) means to rely on
## the standard search path only.
#ssh_privkey: /path/to/private.key