Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

taskotron: update dev to buildbot 1.7, Python3

Browse source 
Merges: https://pagure.io/fedora-qa/qa-ansible/pull-request/6
This commit is contained in:
Kamil Páral 2019-02-05 15:17:01 +01:00
parent a4e367c1f3
commit 758c0d035e
20 changed files with 764 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

2
inventory/group_vars/taskotron-dev
View file

@ -32,7 +32,7 @@ grokmirror_default_branch: develop
############################################################
execdb_endpoint: execdb
execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbottest
execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbot
execdb_server: http://resultsdb-dev01.qa.fedoraproject.org/execdb
vault_endpoint: vault

1
inventory/group_vars/taskotron-dev-client-hosts
View file

@ -24,6 +24,7 @@ slaves_group: buildslaves
buildmaster: 10.5.124.181
buildslave_port: 9989
buildslave_password: "{{ dev_buildslave_password }}"
buildslaves_home: /srv/buildslaves
taskotron_admin_email: taskotron-admin-members@fedoraproject.org

4
roles/taskotron/buildmaster-configure/templates/buildmaster.service.j2
View file

@ -3,7 +3,7 @@ Description=Buildmaster for taskbot
After=network.target
# Workaround for builmaster not starting on system reboot
# https://pagure.io/taskotron/issue/139
StartLimitIntervalSec=30m
StartLimitIntervalSec=15m
StartLimitBurst=5
[Service]
@ -18,7 +18,7 @@ Group={{ buildmaster_user }}
# Workaround for builmaster not starting on system reboot
# https://pagure.io/taskotron/issue/139
Restart=on-failure
RestartSec=60
RestartSec=30
[Install]
WantedBy=multi-user.target

443
roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2.dev Normal file
View file

@ -0,0 +1,443 @@
# -*- python -*-
# ex: set filetype=python:
from buildbot.plugins import *
# This is a buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
c['buildbotNetUsageData'] = 'basic'
####### WORKERS
# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
c['workers'] = [
{% for buildslave in buildslaves %}
{% if deployment_type in ['dev', 'stg', 'prod'] %}
worker.Worker("{{ buildslave }}", "{{ buildslave_password }}"),
{% elif deployment_type == 'local' %}
worker.Worker("{{ buildslave }}", "{{ local_buildslave_password }}"),
{% endif %}
{% endfor %}
]
# 'protocols' contains information about protocols which master will use for
# communicating with workers. You must define at least 'port' option that workers
# could connect to your master with this protocol.
# 'port' must match the value configured into the workers (with their
# --master option)
c['protocols'] = {'pb': {'port': {{ buildslave_port }} }}
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'runtests' build
import buildbot.schedulers.basic
class JobScheduler(buildbot.schedulers.basic.BaseBasicScheduler):
def __init__(self, name, **kwargs):
buildbot.schedulers.basic.BaseBasicScheduler.__init__(self, name, **kwargs)
def getChangeFilter(self, branch, branches, change_filter, categories):
return util.ChangeFilter.fromSchedulerConstructorArgs(
change_filter=change_filter, categories=categories)
def getTimerNameForChange(self, changes):
return "only"
def getChangeClassificationsForTimer(self, objectid, timer_name):
return self.master.db.schedulers.getChangeClassifications(
self.objectid)
c['schedulers'] = []
c['schedulers'].append(JobScheduler(
name="jobsched-noarch",
builderNames=["all"],
treeStableTimer=None,
change_filter=util.ChangeFilter(project='rpmcheck',
category='noarch')))
c['schedulers'].append(JobScheduler(
name="jobsched-i386",
builderNames=['i386'],
treeStableTimer=None,
change_filter=util.ChangeFilter(project='rpmcheck',
category='i386')))
c['schedulers'].append(JobScheduler(
name="jobsched-x86_64",
builderNames=['x86_64'],
treeStableTimer=None,
change_filter=util.ChangeFilter(project='rpmcheck',
category='x86_64')))
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=["all", 'x86_64', 'i386']))
c['schedulers'].append(schedulers.ForceScheduler(
name="rpmcheck",
builderNames=["all", 'x86_64', 'i386'],
properties=[
util.StringParameter(
name='taskname',
label='name of check to run',
default='',
size=256,
),
util.StringParameter(
name='item',
label='object to use',
default='',
size=256,
),
util.StringParameter(
name='item_type',
label='type of object to use',
default='',
size=256,
),
util.StringParameter(
name='uuid',
label='UUID of the build in progress',
default='',
size=256,
),
util.StringParameter(
name='arch',
label='arch of rpm to test',
default='x86_64',
size=128,
),
]))
####### RESOURCE LOCKS
#
# This is a set of resource locks to make sure that we don't have too many things
# going on on each slave at one time.
build_lock = util.WorkerLock("worker_builds",
maxCount=1)
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
factory = util.BuildFactory()
{% if deployment_type in ['local'] %}
# clean out /var/tmp/taskotron (see T253)
factory.addStep(steps.ShellCommand(
command="rm -rf /var/tmp/taskotron/*",
name="rm_tmp",
descriptionDone=['Clean tmp'],
))
# clean the log (see T230)
factory.addStep(steps.ShellCommand(
command=["rm", "-f", "/var/log/taskotron/taskotron.log"],
name="rm_log",
descriptionDone=['Clean log'],
))
{% endif %}
{% if deployment_type in ['dev', 'stg', 'prod'] %}
# clean out /var/tmp/taskotron (see T253)
factory.addStep(steps.ShellCommand(
command=util.Interpolate("rm -rf /var/tmp/taskotron/%(prop:workername)s/*"),
name="rm_tmp",
description='Clean tmp',
))
{% endif %}
# check out the source
factory.addStep(steps.Git(
repourl=util.Property('git_repo', default=util.Interpolate('file:///var/lib/git/mirror/fedoraqa/%(prop:taskname)s/')),
branch=util.Property('git_branch', default='{{ grokmirror_default_branch }}'),
mode='full',
method='clobber',
shallow=True,
descriptionDone='Clone task',
))
# run the runner
factory.addStep(steps.ShellCommand(
command=['sudo', "runtask",
'-i', util.Interpolate('%(prop:item)s'),
'-t', util.Interpolate('%(prop:item_type)s'),
'-a', util.Interpolate('%(prop:arch)s'),
'-j', util.Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'),
'--uuid', util.Interpolate('%(prop:uuid)s'),
'.',
],
description=[
util.Interpolate('%(prop:taskname)s on '),
util.Interpolate('%(prop:item)s (%(prop:arch)s)')
],
name='runtask',
timeout=20*60,
{% if deployment_type in ['dev', 'stg', 'prod'] %}
sigtermTime=5*60,
lazylogfiles=True,
logfiles={
'taskotron.log': {'filename': util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/taskotron.log')},
'heartbeat.log': {'filename': util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/heartbeat.log')},
},
{% elif deployment_type in ['local'] %}
logfiles={
'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log'},
},
{% endif %}
))
# make sure minion is removed
factory.addStep(steps.ShellCommand(
command=util.Interpolate('testcloud instance remove --force taskotron-%(prop:uuid)s; true'),
description='Remove minion',
))
# create artifacts dir on master
factory.addStep(steps.MasterShellCommand(
command=["mkdir", '-m', '0755', util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s')],
descriptionDone='Create artifacs dir',
))
# copy artifacts to master
factory.addStep(steps.DirectoryUpload(
workersrc=util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'),
masterdest=util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/'),
descriptionDone='Copy artifacts',
))
# save stdio from runtask step
# FIXME: worked with buildbot 0.8, later buildbots changed URLs
# factory.addStep(steps.MasterShellCommand(
# command=['curl', '-o', util.Interpolate('/srv/taskotron/artifacts/%(prop:uuid)s/taskotron/stdio.log'),
# util.Interpolate('http://127.0.0.1/taskmaster/builders/%(prop:buildername)s/builds/'
# '%(prop:buildnumber)s/steps/runtask/logs/stdio/text')
# ],
# descriptionDone='Save runtask stdio log',
# ))
# ensure all artifacts are readable
factory.addStep(steps.MasterShellCommand(
command=['chmod', '-R', 'o+r', util.Interpolate('/srv/taskotron/artifacts/%(prop:uuid)s/')],
descriptionDone='Set file permissions',
))
# gzip artifacts
factory.addStep(steps.MasterShellCommand(
command=util.Interpolate('find {{ public_artifacts_dir }}/%(prop:uuid)s/ -type f -exec gzip {} \;'),
descriptionDone='Compress artifacs',
))
{% if deployment_type in ['local'] %}
# copy taskotron log to master
factory.addStep(steps.FileUpload(
workersrc='/var/log/taskotron/taskotron.log',
masterdest=util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'),
mode=0644,
))
{% endif %}
# render current time when needed
import datetime
from buildbot.process.properties import renderer
@renderer
def today(props):
return datetime.datetime.now().strftime("%Y%m%d")
# move the artifacts to the correct dir on the master
factory.addStep(steps.MasterShellCommand(
command=util.Interpolate(
'mkdir -p -m 0755 {{ public_artifacts_dir }}/%(kw:today)s && '
'mkdir -p -m 0755 {{ public_artifacts_dir }}/all && '
'mv {{ public_artifacts_dir }}/%(prop:uuid)s/ {{ public_artifacts_dir }}/%(kw:today)s/ && '
'ln -s {{ public_artifacts_dir }}/%(kw:today)s/%(prop:uuid)s {{ public_artifacts_dir }}/all/',
today=today),
descriptionDone='Move artifacs',
))
c['builders'] = []
c['builders'].append(util.BuilderConfig(
name="x86_64",
workernames=[
{% for buildslave in x86_64_buildslaves %}
"{{ buildslave }}",
{% endfor %}
],
factory=factory,
locks=[
build_lock.access('counting')
],
collapseRequests=False,
))
c['builders'].append(util.BuilderConfig(
name="i386",
workernames=[
{% for buildslave in i386_buildslaves %}
"{{ buildslave }}",
{% endfor %}
],
factory=factory,
locks=[
build_lock.access('counting')
],
collapseRequests=False,
))
c['builders'].append(util.BuilderConfig(
name="all",
workernames=[
{% for buildslave in buildslaves %}
"{{ buildslave }}",
{% endfor %}
],
factory=factory,
locks=[
build_lock.access('counting')
],
collapseRequests=False,
))
####### BUILDBOT SERVICES
# 'services' is a list of BuildbotService items like reporter targets. The
# status of each build will be pushed to these targets. buildbot/reporters/*.py
# has a variety to choose from, like IRC bots.
c['services'] = []
## Email notifications
from buildbot.plugins import reporters
{% raw %}
email_template = u'''
<h4>Build status: {{ summary }}</h4>
<table cellspacing="10">
<tr><td>Worker for this Build:</td><td><b>{{ workername }}</b></td></tr>
<tr><td>Complete logs:</td><td><a href="{{ build_url }}">{{ build_url }}</a></td></tr>
</table>
{% for step in build['steps'] if statuses[step.results] in ["warnings", "failure", "exception"] %}
<p><b>Step: {{ step['name'] }}: {{ statuses[step['results']] }}</b></p>
{% for log in step['logs'] %}
<p>Last 40 lines of <i>{{ log['name'] }}</i>:</p>
{# buildbot returns an extra first character for each line (a bug?) denoting its color #}
<pre>
{% for line in log['content']['content'].split('\n')[-40:] %}{{ line[1:] }}
{% endfor %}
</pre>
{% endfor %}
<hr/>
{% endfor %}
'''
{% endraw %}
html_message_formatter = reporters.MessageFormatter(
template=email_template,
template_type='html',
wantProperties=True,
wantSteps=True,
wantLogs=True,
ctx=dict(statuses=util.Results),
)
mn = reporters.MailNotifier(
fromaddr='taskotron@fedoraproject.org',
sendToInterestedUsers=False,
subject="XXX %(projectName)s %(result)s on {{ deployment_type }} %(builder)s",
mode=('failing', 'exception', 'warnings'),
extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'],
relayhost="bastion.phx2.fedoraproject.org",
messageFormatter=html_message_formatter)
c['services'].append(mn)
## ExecDB's push-notifications
sp = reporters.HttpStatusPush(
serverUrl="{{ execdb_statuspush }}",
wantProperties=True,
wantSteps=True,
)
c['services'].append(sp)
####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot installation's
# home pages (linked to the 'titleURL').
c['title'] = "Taskotron"
c['titleURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/"
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server is visible. This typically uses the port number set in
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
c['buildbotURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/"
####### WEB
c['www'] = {
'port': 8010,
'plugins': {
'waterfall_view': {},
'console_view': {},
'grid_view': {},
},
'change_hook_dialects': {
'base': True,
},
'auth': util.UserPasswordAuth({
{% if deployment_type == 'dev' %}
"{{ dev_buildbot_user }}": "{{ dev_buildbot_password }}",
{% elif deployment_type == 'stg' %}
"{{ stg_buildbot_user }}": "{{ stg_buildbot_password }}",
{% elif deployment_type == 'prod' %}
"{{ prod_buildbot_user }}": "{{ prod_buildbot_password }}",
{% elif deployment_type == 'local' %}
"{{ local_buildbot_user }}": "{{ local_buildbot_password }}",
{% endif %}
}),
}
####### DB URL
c['db'] = {
# This specifies what database buildbot uses to store its state.
{% if deployment_type in ['dev', 'stg', 'prod'] %}
'db_url' : "postgresql://{{ buildmaster_db_user }}:{{ buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}",
{% elif deployment_type == 'local' %}
'db_url' : "postgresql://{{ local_buildmaster_db_user }}:{{ local_buildmaster_db_password }}@127.0.0.1/{{ buildmaster_db_name }}",
{% endif %}
}
# TODO: REMOVE THIS
c['db'] = {
'db_url' : "sqlite:///state.sqlite",
}

36
roles/taskotron/buildmaster/tasks/main.yml
View file

@ -2,17 +2,20 @@
- name: start httpd (provided in the apache role)
service: name=httpd state=started
- name: ensure packages required for buildmaster are installed (yum)
package: name={{ item }} state=present
- name: ensure packages required for buildmaster are installed
dnf: name={{ item }} state=present
with_items:
- buildbot
- buildbot-master
- libsemanage-python
- python-psycopg2
- buildbot-www
- python2-libsemanage
- python3-psycopg2
- policycoreutils-python-utils
when: ansible_distribution_major_version|int < 22
- python3-treq
when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
- name: ensure packages required for buildmaster are installed (dnf)
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for buildmaster are installed
dnf: name={{ item }} state=present
with_items:
- buildbot
@ -20,17 +23,12 @@
- libsemanage-python
- python-psycopg2
- policycoreutils-python-utils
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
- name: ensure additional packages required for CI buildmaster are installed
package: name={{ item }} state=present
when: ansible_distribution_major_version|int < 22 and (deployment_type == 'qa-prod' or deployment_type == 'qa-stg')
with_items:
- python-novaclient
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
############
- name: ensure additional packages required for CI buildmaster are installed
dnf: name={{ item }} state=present
when: ansible_distribution_major_version|int > 21 and (deployment_type == 'qa-prod' or deployment_type == 'qa-stg')
when: deployment_type in ['qa-prod', 'qa-stg']
with_items:
- python-novaclient
@ -50,7 +48,7 @@
when: deployment_type in ['qa-stg']
- name: ensure correct fcontext for buildmaster home (new)
file: path="{{ buildmaster_home }}(/. *)?" setype=var_lib_t owner=buildmaster group=buildmaster mode=0775 state=directory
file: path="{{ buildmaster_home }}" setype=var_lib_t owner=buildmaster group=buildmaster mode=0775 state=directory
when: deployment_type in ['dev', 'stg', 'prod']
#- name: make sure the selinux fcontext is restored
@ -65,6 +63,12 @@
notify:
- reload httpd
- name: ensure postgresql service is running
systemd:
name: postgresql
state: started
enabled: true
- name: ensure buildmaster database is created
when: buildmaster_db_host != 'localhost'
delegate_to: "{{ buildmaster_db_host }}"

13
roles/taskotron/buildmaster/templates/buildmaster.conf.j2
View file

@ -1,4 +1,13 @@
<Location /{{ buildmaster_endpoint }}/ >
ProxyPass http://127.0.0.1:8010/
ProxyPassReverse http://127.0.0.1:8010/
ProxyPass "http://127.0.0.1:8010/"
ProxyPassReverse "http://127.0.0.1:8010/"
ProxyPreserveHost On
</Location>
<Location /{{ buildmaster_endpoint }}/ws >
ProxyPass "ws://127.0.0.1:8010/ws"
ProxyPassReverse "ws://127.0.0.1:8010/ws"
ProxyPreserveHost On
</Location>

4
roles/taskotron/buildslave-configure/defaults/main.yml
View file

@ -1,2 +1,6 @@
---
taskotron_admin_email: root@localhost.localdomain
slaves:
- user: ""
home: ""
dir: ""

55
roles/taskotron/buildslave-configure/tasks/main.yml
View file

@ -7,7 +7,10 @@
- name: generate slave config
become: true
become_user: "{{ slave_user }}"
template: src=buildbot.tac.j2 dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
template: src={{ item }} dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
with_first_found:
- 'buildbot.tac.j2.{{ deployment_type }}'
- buildbot.tac.j2
when: deployment_type in ['local', 'qa-stg']
- name: generate slave info
@ -37,7 +40,10 @@
lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }}
- name: generate buildslave service file
template: src=buildslave.service.j2 dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
template: src={{ item }} dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
with_first_found:
- 'buildslave.service.j2.{{ deployment_type }}'
- buildslave.service.j2
when: deployment_type in ['local', 'qa-stg']
register: buildslave_service
@ -49,20 +55,38 @@
service: name=buildslave enabled=yes state={{ (buildslave_service.changed) | ternary('restarted','started') }}
when: deployment_type in ['local', 'qa-stg']
- name: create slave
become: true
become_user: '{{ item.user }}'
command: creates={{ item.dir }} buildbot-worker create-worker {{ item.dir }} {{ buildmaster }} buildslave passwd
with_items:
- '{{ slaves }}'
when: deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: create slave
become: true
become_user: '{{ item.user }}'
command: creates={{ item.dir }} buildslave create-slave {{ item.dir }} {{ buildmaster }} buildslave passwd
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
when: deployment_type in ['dev', 'stg', 'prod']
when: deployment_type in ['stg', 'prod']
############
- name: figure out which buildbot.tac.j2 file to use
set_fact: buildbot_tac_j2_file={{ item | basename }}
# this unfortunately searches in files/
with_first_found:
- '../templates/buildbot.tac.j2.{{ deployment_type }}'
- '../templates/buildbot.tac.j2'
- name: generate slave config
become: true
become_user: '{{ item.user }}'
template: src=buildbot.tac.j2 dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
template: src={{ buildbot_tac_j2_file }} dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: deployment_type in ['dev', 'stg', 'prod']
- name: generate slave admin info
@ -70,7 +94,7 @@
become_user: '{{ item.user }}'
template: src=admin.j2 dest={{ item.dir }}/info/admin mode=0644 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: deployment_type in ['dev', 'stg', 'prod']
- name: generate slave host info
@ -78,41 +102,44 @@
become_user: '{{ item.user }}'
template: src=host.j2 dest={{ item.dir }}/info/host mode=0644 owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: deployment_type in ['dev', 'stg', 'prod']
- name: create slave ssh directory
file: path={{ item.home }}/.ssh owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
- name: install slave ssh private key
copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa owner={{ item.user }} group={{ slaves_group }} mode=0600
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: buildslave_private_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
- name: install slave ssh public key
copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa.pub owner={{ item.user }} group={{ slaves_group }} mode=0644
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
- name: make sure master is in known_hosts
lineinfile: dest={{ item.home }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ item.user }} group={{ slaves_group }}
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
- name: ensure correct fcontext for buildslave dir
file: path={{ item.home }}/slave setype=var_lib_t owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: deployment_type in ['dev', 'stg', 'prod']
- name: generate buildslave service file
template: src=buildslave@.service.j2 dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0644
template: src={{ item }} dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0644
with_first_found:
- 'buildslave@.service.j2.{{ deployment_type }}'
- buildslave@.service.j2
when: deployment_type in ['dev', 'stg', 'prod']
register: buildslave_service
@ -123,5 +150,5 @@
- name: start and enable buildslave services
service: name=buildslave@{{ item.user }} enabled=yes state={{ (buildslave_service.changed) | ternary('restarted','started') }}
with_items:
- '{{ slaves|default([dict(user="", home="", dir="")]) }}'
- '{{ slaves }}'
when: deployment_type in ['dev', 'stg', 'prod']

53
roles/taskotron/buildslave-configure/templates/buildbot.tac.j2.dev Normal file
View file

@ -0,0 +1,53 @@
import os
from buildbot_worker.bot import Worker
from twisted.application import service
{% if deployment_type in ['local', 'qa-stg'] %}
basedir = '/home/buildslave/slave'
{% endif %}
{% if deployment_type in ['dev', 'stg', 'prod'] %}
basedir = '{{ item.dir }}'
{% endif %}
rotateLength = 10000000
maxRotatedFiles = 10
# if this is a relocatable tac file, get the directory containing the TAC
if basedir == '.':
import os.path
basedir = os.path.abspath(os.path.dirname(__file__))
# note: this line is matched against to check that this is a worker
# directory; do not edit it.
application = service.Application('buildbot-worker')
from twisted.python.logfile import LogFile
from twisted.python.log import ILogObserver, FileLogObserver
logfile = LogFile.fromFullPath(
os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
maxRotatedFiles=maxRotatedFiles)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
buildmaster_host = '{{ buildmaster }}'
port = 9989
{% if deployment_type in ['local', 'qa-stg', 'qa-prod'] %}
workername = '{{ buildslave_name }}'
passwd = '{{ buildslave_password }}'
{% endif %}
{% if deployment_type in ['dev', 'stg', 'prod'] %}
workername = '{{ item.user }}'
passwd = '{{ buildslave_password }}'
{% endif %}
keepalive = 600
umask = 0o0022
maxdelay = 300
numcpus = None
allow_shutdown = None
maxretries = None
s = Worker(buildmaster_host, port, workername, passwd, basedir,
keepalive, umask=umask, maxdelay=maxdelay,
numcpus=numcpus, allow_shutdown=allow_shutdown,
maxRetries=maxretries)
s.setServiceParent(application)

15
roles/taskotron/buildslave-configure/templates/buildslave.service.j2.dev Normal file
View file

@ -0,0 +1,15 @@
[Unit]
Description=Buildslave for taskbot
After=network.target
[Service]
Type=forking
# disabled because of https://pagure.io/taskotron/issue/236
#PIDFile=/home/buildslave/slave/twistd.pid
ExecStart=/usr/bin/buildbot-worker start /home/buildslave/slave/
ExecStop=/usr/bin/buildbot-worker stop /home/buildslave/slave/
User=buildslave
Group=buildslave
[Install]
WantedBy=multi-user.target

15
roles/taskotron/buildslave-configure/templates/buildslave@.service.j2.dev Normal file
View file

@ -0,0 +1,15 @@
[Unit]
Description=Buildslave for taskotron
After=network.target
[Service]
Type=forking
# disabled because of https://pagure.io/taskotron/issue/236
#PIDFile=/home/%i/slave/twistd.pid
ExecStart=/usr/bin/buildbot-worker start /srv/buildslaves/%i/slave/
ExecStop=/usr/bin/buildbot-worker stop /srv/buildslaves/%i/slave/
User=%i
Group={{ slaves_group }}
[Install]
WantedBy=multi-user.target

52
roles/taskotron/buildslave/tasks/main.yml
View file

@ -1,9 +1,33 @@
---
- name: ensure packages required for buildslave are installed (dnf)
- name: ensure packages required for buildslave are installed
dnf: name={{ item }} state=present
with_items:
- buildbot-worker
- git
- policycoreutils-python-utils
when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for buildslave are installed
dnf: name={{ item }} state=present
with_items:
- buildbot-slave
- git
- policycoreutils-python-utils
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
############
- name: ensure packages required for CI buildslave are installed
dnf: name={{ item }} state=present
with_items:
- testcloud
- mash
- mock
- koji
- python-fedora
- python-doit
- python2-rpmfluff
- rpmlint
- koji
- bodhi-client
@ -14,20 +38,7 @@
- pylint
- python-pep8
- moreutils
- policycoreutils-python-utils
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
- name: ensure packages required for CI buildslave are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- testcloud
- mash
- mock
- koji
- python-fedora
- python-doit
- python2-rpmfluff
when: (deployment_type == 'qa-stg') and (ansible_distribution_major_version|int > 21) and (ansible_cmdline.ostree is not defined)
when: (deployment_type in ['qa-stg']) and (ansible_cmdline.ostree is not defined)
- name: set the selinux fcontext type for the buildmaster_home to var_lib_t
command: semanage fcontext -a -t var_lib_t "{{ slave_dir }}(/.*)?"
@ -52,6 +63,13 @@
group: name={{ slaves_group }}
when: slaves_group is defined and (deployment_type in ['dev', 'stg', 'prod'])
- name: create buildslaves home
file:
path: "{{ buildslaves_home }}"
state: directory
mode: 0755
when: slaves is defined and (deployment_type in ['dev', 'stg', 'prod'])
- name: add buildslave users
user: name={{ item.user }} group={{ slaves_group }} groups=testcloud,taskotron home={{ item.home }}
with_items: "{{ slaves }}"
@ -63,13 +81,13 @@
when: slaves is defined and deployment_type in ['qa-prod', 'qa-stg']
- name: make sure the selinux fcontext is restored
command: restorecon -R "{{ item.dir }}"
command: restorecon -R "{{ item.home }}"
with_items: "{{ slaves }}"
when: slaves is defined and deployment_type in ['dev', 'stg', 'prod', 'qa-prod', 'qa-stg']
- name: disable ansible host key checking
lineinfile:
dest: /etc/ansible/ansible.cfg
regexp: '^#host_key_checking = False$'
regexp: '^#?host_key_checking = False$'
line: 'host_key_checking = False'
when: deployment_type in ['dev']

17
roles/taskotron/execdb/tasks/main.yml
View file

@ -2,15 +2,17 @@
- name: start httpd (provided in the apache role)
service: name=httpd state=started
- name: ensure packages required for execdb are installed (yum)
package: name={{ item }} state=present
- name: ensure packages required for execdb are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- execdb
- mod_wsgi
- python-psycopg2
- libsemanage-python
when: ansible_distribution_major_version|int < 22
- python3-mod_wsgi
- python3-psycopg2
- python2-libsemanage
when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for execdb are installed (dnf)
dnf: name={{ item }} state=present
with_items:
@ -18,7 +20,8 @@
- mod_wsgi
- python-psycopg2
- libsemanage-python
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
###########
- name: ensure database is created
delegate_to: "{{ execdb_db_host_machine }}"

23
roles/taskotron/resultsdb-backend/tasks/main.yml
View file

@ -2,23 +2,26 @@
- name: start httpd (provided in the apache role)
service: name=httpd state=started
- name: ensure packages required for resultsdb are installed (yum)
package: name={{ item }} state=present
when: ansible_distribution_major_version|int < 22
with_items:
- resultsdb
- mod_wsgi
- python-psycopg2
- libsemanage-python
- name: ensure packages required for resultsdb are installed (dnf)
dnf: name={{ item }} state=present
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
with_items:
- resultsdb
- python3-mod_wsgi
- python3-psycopg2
- python2-libsemanage
when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for resultsdb are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- resultsdb
- mod_wsgi
- python-psycopg2
- libsemanage-python
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
###########
- name: ensure database is created
delegate_to: "{{ resultsdb_db_host_machine }}"

19
roles/taskotron/resultsdb-frontend/tasks/main.yml
View file

@ -2,19 +2,22 @@
- name: start httpd (provided in the apache role)
service: name=httpd state=started
- name: ensure packages required for resultsdb_frontend are installed (yum)
package: name={{ item }} state=present
when: ansible_distribution_major_version|int < 22
with_items:
- resultsdb_frontend
- mod_wsgi
- name: ensure packages required for resultsdb_frontend are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- resultsdb_frontend
- python3-mod_wsgi
when: ansible_cmdline.ostree is not defined and and deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for resultsdb_frontend are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- resultsdb_frontend
- mod_wsgi
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
###########
- name: ensure selinux lets httpd talk to the network
seboolean: name=httpd_can_network_connect persistent=yes state=yes

23
roles/taskotron/taskotron-client/tasks/main.yml
View file

@ -1,12 +1,12 @@
---
- name: ensure packages required for libtaskotron are installed (yum)
package: name={{ item }} state=present
- name: ensure packages required for libtaskotron are installed
dnf: name={{ item }} state=present
with_items:
- PyYAML
- libtaskotron
when: deployment_type == 'local'
- name: ensure packages required for libtaskotron are installed (dnf)
- name: ensure packages required for libtaskotron are installed
dnf: name={{ item }} state=present
with_items:
- PyYAML
@ -15,21 +15,13 @@
- selinux-policy-devel
when: deployment_type in ['dev', 'stg', 'prod']
- name: ensure packages required for taskotron tasks are installed (yum)
package: name={{ item }} state=present
with_items:
- python-solv
- python-librepo
- rpmlint
when: ansible_distribution_major_version|int < 22
- name: ensure packages required for taskotron tasks are installed (dnf)
- name: ensure packages required for taskotron tasks are installed
dnf: name={{ item }} state=present
with_items:
- python-solv
- python-librepo
- rpmlint
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
when: ansible_cmdline.ostree is not defined
- name: generate taskotron.yaml config file
template: src={{ item }} dest=/etc/taskotron/taskotron.yaml owner=root group=root mode=0644
@ -109,16 +101,11 @@
command: chdir=/root/ semodule -i /root/service-virt-transition.pp
when: deployment_type in ['dev', 'stg', 'prod']
- name: enable libvirtd
service: name=libvirtd state=started enabled=yes
when: ansible_distribution_major_version|int < 24
- name: start and enable libvirt services
service: name={{ item }} state=started enabled=yes
with_items:
- libvirtd
- virtlogd
when: ansible_distribution_major_version|int >= 24
- name: copy custom libvirt network config
copy: src=default.xml dest=/etc/libvirt/qemu/networks/default.xml owner=root group=root mode=0600

68
roles/taskotron/taskotron-trigger/tasks/main.yml
View file

@ -1,23 +1,13 @@
---
- name: ensure packages required for taskotron-trigger are installed (yum)
package: name={{ item }} state=present
with_items:
- fedmsg
- fedmsg-hub
- taskotron-trigger
# needed for cron script
- python-resultsdb_api
when: ansible_distribution_major_version|int < 22
- name: ensure packages required for taskotron-trigger are installed (dnf)
- name: ensure packages required for taskotron-trigger are installed
dnf: name={{ item }} state=present
with_items:
- fedmsg
- fedmsg-hub
- taskotron-trigger
# needed for cron script
- python2-resultsdb_api
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
- python2-resultsdb_api # TODO: remove when transition to buildbot 1.7 is complete
- python3-resultsdb_api
when: ansible_cmdline.ostree is not defined
- name: generate trigger configuration
template: src={{ item }} dest=/etc/taskotron/trigger.cfg owner=root group=root mode=0744
@ -25,7 +15,20 @@
- trigger.cfg.j2.{{ deployment_type }}
- trigger.cfg.j2
notify:
- restart fedmsg-hub
- restart fedmsg-hub-3
when: deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: generate trigger configuration
template: src={{ item }} dest=/etc/taskotron/trigger.cfg owner=root group=root mode=0744
with_first_found:
- trigger.cfg.j2.{{ deployment_type }}
- trigger.cfg.j2
notify:
- restart fedmsg-hub
when: deployment_type in ['stg', 'prod']
###########
- name: generate trigger rules
template: src={{ item }} dest=/etc/taskotron/trigger_rules.yml owner=root group=root mode=0744
@ -33,12 +36,35 @@
- trigger_rules.yml.j2.{{ deployment_type }}
- trigger_rules.yml.j2
notify:
- restart fedmsg-hub
- restart fedmsg-hub-3
when: deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: generate trigger rules
template: src={{ item }} dest=/etc/taskotron/trigger_rules.yml owner=root group=root mode=0744
with_first_found:
- trigger_rules.yml.j2.{{ deployment_type }}
- trigger_rules.yml.j2
notify:
- restart fedmsg-hub
when: deployment_type in ['stg', 'prod']
#########
- name: copy fedmsg logging configuration
copy: src=logging.py dest=/etc/fedmsg.d/logging.py owner=root group=root mode=0744
notify:
- restart fedmsg-hub
- restart fedmsg-hub-3
when: deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: copy fedmsg logging configuration
copy: src=logging.py dest=/etc/fedmsg.d/logging.py owner=root group=root mode=0744
notify:
- restart fedmsg-hub
when: deployment_type in ['stg', 'prod']
###########
- name: install fetch_activity
copy: src=fetch_activity.py dest=/root/fetch_activity.py owner=root group=root mode=0644
@ -54,5 +80,13 @@
- name: copy tmpfiles.d for trigger distgit cache
template: src=tmpfiles.d.trigger.conf.j2 dest=/etc/tmpfiles.d/trigger.conf owner=root group=root mode=0644
- name: start and enable fedmsg-hub-3 service
service: name=fedmsg-hub-3 enabled=yes state=started
when: deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: start and enable fedmsg-hub service
service: name=fedmsg-hub enabled=yes state=started
when: deployment_type in ['stg', 'prod']
###########

1
roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2
View file

@ -21,4 +21,3 @@ deployment_type = stg
[koji_build_completed]
; critpath_filepath = {{ trigger_critpath_file }}
critpath_filepath =

23
roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2.dev Normal file
View file

@ -0,0 +1,23 @@
[buildbot]
url = http://127.0.0.1:8010/change_hook
[trigger]
valid_arches = x86_64,armhfp
koji_url = http://koji.fedoraproject.org/kojihub
datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw
execdb_server = {{ execdb_server }}
runner_type = BuildbotRunner
job_logging = True
joblog_file = {{ trigger_joblog_file }}
fuse_delay = 900 ; 15 minutes in seconds
git_cache_dir = {{ trigger_cache_dir }}
rules_template = {{ trigger_rules_template_path }}
{# enable this if you want taskotron-stg to react to just stg fedmsgs
{% if deployment_type in ['stg'] %}
deployment_type = stg
{% endif%}
#}
[koji_build_completed]
; critpath_filepath = {{ trigger_critpath_file }}
critpath_filepath =

17
roles/taskotron/vault/tasks/main.yml
View file

@ -2,15 +2,17 @@
- name: start httpd (provided in the apache role)
service: name=httpd state=started
- name: ensure packages required for vault are installed (yum)
package: name={{ item }} state=present
- name: ensure packages required for vault are installed (dnf)
dnf: name={{ item }} state=present
with_items:
- vault
- mod_wsgi
- python-psycopg2
- libsemanage-python
when: ansible_distribution_major_version|int < 22
- python3-mod_wsgi
- python3-psycopg2
- python2-libsemanage
when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
###########
## TODO: remove when transition to buildbot 1.7 is complete
- name: ensure packages required for vault are installed (dnf)
dnf: name={{ item }} state=present
with_items:
@ -18,7 +20,8 @@
- mod_wsgi
- python-psycopg2
- libsemanage-python
when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
###########
- name: ensure database is created
delegate_to: "{{ vault_db_host_machine }}"