taskotron: update dev to buildbot 1.7, Python3

Merges: https://pagure.io/fedora-qa/qa-ansible/pull-request/6

inventory/group_vars/taskotron-dev

@@ -32,7 +32,7 @@ grokmirror_default_branch: develop
 ############################################################
 
 execdb_endpoint: execdb
-execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbottest
+execdb_statuspush: http://resultsdb-dev01.qa.fedoraproject.org/execdb/buildbot
 execdb_server: http://resultsdb-dev01.qa.fedoraproject.org/execdb
 
 vault_endpoint: vault

inventory/group_vars/taskotron-dev-client-hosts

@@ -24,6 +24,7 @@ slaves_group: buildslaves
 buildmaster: 10.5.124.181
 buildslave_port: 9989
 buildslave_password: "{{ dev_buildslave_password }}"
+buildslaves_home: /srv/buildslaves
 taskotron_admin_email: taskotron-admin-members@fedoraproject.org
 
 

roles/taskotron/buildmaster-configure/templates/buildmaster.service.j2

@@ -3,7 +3,7 @@ Description=Buildmaster for taskbot
 After=network.target
 # Workaround for builmaster not starting on system reboot
 # https://pagure.io/taskotron/issue/139
-StartLimitIntervalSec=30m
+StartLimitIntervalSec=15m
 StartLimitBurst=5
 
 [Service]
@@ -18,7 +18,7 @@ Group={{ buildmaster_user }}
 # Workaround for builmaster not starting on system reboot
 # https://pagure.io/taskotron/issue/139
 Restart=on-failure
-RestartSec=60
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target

roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2.dev

@@ -0,0 +1,443 @@
+# -*- python -*-
+# ex: set filetype=python:
+
+from buildbot.plugins import *
+
+# This is a buildmaster config file. It must be installed as
+# 'master.cfg' in your buildmaster's base directory.
+
+# This is the dictionary that the buildmaster pays attention to. We also use
+# a shorter alias to save typing.
+c = BuildmasterConfig = {}
+
+c['buildbotNetUsageData'] = 'basic'
+
+
+####### WORKERS
+
+# The 'workers' list defines the set of recognized workers. Each element is
+# a Worker object, specifying a unique worker name and password.  The same
+# worker name and password must be configured on the worker.
+c['workers'] = [
+{% for buildslave in buildslaves %}
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+    worker.Worker("{{ buildslave }}", "{{ buildslave_password }}"),
+{% elif deployment_type == 'local' %}
+    worker.Worker("{{ buildslave }}", "{{ local_buildslave_password }}"),
+{% endif %}
+{% endfor %}
+]
+
+# 'protocols' contains information about protocols which master will use for
+# communicating with workers. You must define at least 'port' option that workers
+# could connect to your master with this protocol.
+# 'port' must match the value configured into the workers (with their
+# --master option)
+c['protocols'] = {'pb': {'port': {{ buildslave_port }} }}
+
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming changes.  In this
+# case, just kick off a 'runtests' build
+
+import buildbot.schedulers.basic
+
+class JobScheduler(buildbot.schedulers.basic.BaseBasicScheduler):
+
+    def __init__(self, name, **kwargs):
+        buildbot.schedulers.basic.BaseBasicScheduler.__init__(self, name, **kwargs)
+
+    def getChangeFilter(self, branch, branches, change_filter, categories):
+        return util.ChangeFilter.fromSchedulerConstructorArgs(
+                change_filter=change_filter, categories=categories)
+
+    def getTimerNameForChange(self, changes):
+        return "only"
+
+    def getChangeClassificationsForTimer(self, objectid, timer_name):
+        return self.master.db.schedulers.getChangeClassifications(
+                self.objectid)
+
+
+c['schedulers'] = []
+c['schedulers'].append(JobScheduler(
+    name="jobsched-noarch",
+    builderNames=["all"],
+    treeStableTimer=None,
+    change_filter=util.ChangeFilter(project='rpmcheck',
+                                    category='noarch')))
+
+c['schedulers'].append(JobScheduler(
+    name="jobsched-i386",
+    builderNames=['i386'],
+    treeStableTimer=None,
+    change_filter=util.ChangeFilter(project='rpmcheck',
+                                    category='i386')))
+
+c['schedulers'].append(JobScheduler(
+    name="jobsched-x86_64",
+    builderNames=['x86_64'],
+    treeStableTimer=None,
+    change_filter=util.ChangeFilter(project='rpmcheck',
+                                    category='x86_64')))
+
+c['schedulers'].append(schedulers.ForceScheduler(
+    name="force",
+    builderNames=["all", 'x86_64', 'i386']))
+
+c['schedulers'].append(schedulers.ForceScheduler(
+    name="rpmcheck",
+    builderNames=["all", 'x86_64', 'i386'],
+    properties=[
+        util.StringParameter(
+            name='taskname',
+            label='name of check to run',
+            default='',
+            size=256,
+        ),
+        util.StringParameter(
+            name='item',
+            label='object to use',
+            default='',
+            size=256,
+        ),
+        util.StringParameter(
+            name='item_type',
+            label='type of object to use',
+            default='',
+            size=256,
+        ),
+        util.StringParameter(
+            name='uuid',
+            label='UUID of the build in progress',
+            default='',
+            size=256,
+        ),
+        util.StringParameter(
+            name='arch',
+            label='arch of rpm to test',
+            default='x86_64',
+            size=128,
+        ),
+    ]))
+
+
+####### RESOURCE LOCKS
+#
+# This is a set of resource locks to make sure that we don't have too many things
+# going on on each slave at one time.
+
+build_lock = util.WorkerLock("worker_builds",
+                             maxCount=1)
+
+
+####### BUILDERS
+
+# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+# what steps, and which workers can execute them.  Note that any particular build will
+# only take place on one worker.
+
+factory = util.BuildFactory()
+
+
+{% if deployment_type in ['local'] %}
+# clean out /var/tmp/taskotron (see T253)
+factory.addStep(steps.ShellCommand(
+    command="rm -rf /var/tmp/taskotron/*",
+    name="rm_tmp",
+    descriptionDone=['Clean tmp'],
+))
+
+# clean the log (see T230)
+factory.addStep(steps.ShellCommand(
+    command=["rm", "-f", "/var/log/taskotron/taskotron.log"],
+    name="rm_log",
+    descriptionDone=['Clean log'],
+))
+{% endif %}
+
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+# clean out /var/tmp/taskotron (see T253)
+factory.addStep(steps.ShellCommand(
+    command=util.Interpolate("rm -rf /var/tmp/taskotron/%(prop:workername)s/*"),
+    name="rm_tmp",
+    description='Clean tmp',
+))
+{% endif %}
+
+# check out the source
+factory.addStep(steps.Git(
+    repourl=util.Property('git_repo', default=util.Interpolate('file:///var/lib/git/mirror/fedoraqa/%(prop:taskname)s/')),
+    branch=util.Property('git_branch', default='{{ grokmirror_default_branch }}'),
+    mode='full',
+    method='clobber',
+    shallow=True,
+    descriptionDone='Clone task',
+))
+
+# run the runner
+factory.addStep(steps.ShellCommand(
+    command=['sudo', "runtask",
+             '-i', util.Interpolate('%(prop:item)s'),
+             '-t', util.Interpolate('%(prop:item_type)s'),
+             '-a', util.Interpolate('%(prop:arch)s'),
+             '-j', util.Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'),
+             '--uuid', util.Interpolate('%(prop:uuid)s'),
+             '.',
+             ],
+    description=[
+        util.Interpolate('%(prop:taskname)s on '),
+        util.Interpolate('%(prop:item)s (%(prop:arch)s)')
+    ],
+    name='runtask',
+    timeout=20*60,
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+    sigtermTime=5*60,
+    lazylogfiles=True,
+    logfiles={
+        'taskotron.log': {'filename': util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/taskotron.log')},
+        'heartbeat.log': {'filename': util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/taskotron/heartbeat.log')},
+    },
+{% elif deployment_type in ['local'] %}
+    logfiles={
+        'taskotron.log': {'filename': '/var/log/taskotron/taskotron.log'},
+    },
+{% endif %}
+))
+
+# make sure minion is removed
+factory.addStep(steps.ShellCommand(
+    command=util.Interpolate('testcloud instance remove --force taskotron-%(prop:uuid)s; true'),
+    description='Remove minion',
+))
+
+# create artifacts dir on master
+factory.addStep(steps.MasterShellCommand(
+    command=["mkdir", '-m', '0755', util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s')],
+    descriptionDone='Create artifacs dir',
+))
+
+# copy artifacts to master
+factory.addStep(steps.DirectoryUpload(
+    workersrc=util.Interpolate('/var/lib/taskotron/artifacts/%(prop:uuid)s/'),
+    masterdest=util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/'),
+    descriptionDone='Copy artifacts',
+))
+
+# save stdio from runtask step
+# FIXME: worked with buildbot 0.8, later buildbots changed URLs
+# factory.addStep(steps.MasterShellCommand(
+#     command=['curl', '-o', util.Interpolate('/srv/taskotron/artifacts/%(prop:uuid)s/taskotron/stdio.log'),
+#              util.Interpolate('http://127.0.0.1/taskmaster/builders/%(prop:buildername)s/builds/'
+#                               '%(prop:buildnumber)s/steps/runtask/logs/stdio/text')
+#     ],
+#     descriptionDone='Save runtask stdio log',
+# ))
+
+# ensure all artifacts are readable
+factory.addStep(steps.MasterShellCommand(
+    command=['chmod', '-R', 'o+r', util.Interpolate('/srv/taskotron/artifacts/%(prop:uuid)s/')],
+    descriptionDone='Set file permissions',
+))
+
+# gzip artifacts
+factory.addStep(steps.MasterShellCommand(
+    command=util.Interpolate('find {{ public_artifacts_dir }}/%(prop:uuid)s/ -type f -exec gzip {} \;'),
+    descriptionDone='Compress artifacs',
+))
+
+{% if deployment_type in ['local'] %}
+# copy taskotron log to master
+factory.addStep(steps.FileUpload(
+    workersrc='/var/log/taskotron/taskotron.log',
+    masterdest=util.Interpolate('{{ public_artifacts_dir }}/%(prop:uuid)s/taskotron.log'),
+    mode=0644,
+))
+{% endif %}
+
+# render current time when needed
+import datetime
+from buildbot.process.properties import renderer
+
+@renderer
+def today(props):
+    return datetime.datetime.now().strftime("%Y%m%d")
+
+# move the artifacts to the correct dir on the master
+factory.addStep(steps.MasterShellCommand(
+    command=util.Interpolate(
+        'mkdir -p -m 0755 {{ public_artifacts_dir }}/%(kw:today)s && '
+        'mkdir -p -m 0755 {{ public_artifacts_dir }}/all && '
+        'mv {{ public_artifacts_dir }}/%(prop:uuid)s/ {{ public_artifacts_dir }}/%(kw:today)s/ && '
+        'ln -s {{ public_artifacts_dir }}/%(kw:today)s/%(prop:uuid)s {{ public_artifacts_dir }}/all/',
+        today=today),
+    descriptionDone='Move artifacs',
+))
+
+
+c['builders'] = []
+c['builders'].append(util.BuilderConfig(
+    name="x86_64",
+    workernames=[
+{% for buildslave in x86_64_buildslaves %}
+        "{{ buildslave }}",
+{% endfor %}
+    ],
+    factory=factory,
+    locks=[
+        build_lock.access('counting')
+    ],
+    collapseRequests=False,
+))
+
+c['builders'].append(util.BuilderConfig(
+    name="i386",
+    workernames=[
+{% for buildslave in i386_buildslaves %}
+        "{{ buildslave }}",
+{% endfor %}
+    ],
+    factory=factory,
+    locks=[
+        build_lock.access('counting')
+    ],
+    collapseRequests=False,
+))
+
+c['builders'].append(util.BuilderConfig(
+    name="all",
+    workernames=[
+{% for buildslave in buildslaves %}
+        "{{ buildslave }}",
+{% endfor %}
+    ],
+    factory=factory,
+    locks=[
+        build_lock.access('counting')
+    ],
+    collapseRequests=False,
+))
+
+
+####### BUILDBOT SERVICES
+
+# 'services' is a list of BuildbotService items like reporter targets. The
+# status of each build will be pushed to these targets. buildbot/reporters/*.py
+# has a variety to choose from, like IRC bots.
+
+c['services'] = []
+
+## Email notifications
+
+from buildbot.plugins import reporters
+
+{% raw %}
+email_template = u'''
+<h4>Build status: {{ summary }}</h4>
+<table cellspacing="10">
+<tr><td>Worker for this Build:</td><td><b>{{ workername }}</b></td></tr>
+<tr><td>Complete logs:</td><td><a href="{{ build_url }}">{{ build_url }}</a></td></tr>
+</table>
+{% for step in build['steps'] if statuses[step.results] in ["warnings", "failure", "exception"] %}
+<p><b>Step: {{ step['name'] }}: {{ statuses[step['results']] }}</b></p>
+{% for log in step['logs'] %}
+<p>Last 40 lines of <i>{{ log['name'] }}</i>:</p>
+{# buildbot returns an extra first character for each line (a bug?) denoting its color #}
+<pre>
+{% for line in log['content']['content'].split('\n')[-40:] %}{{ line[1:] }}
+{% endfor %}
+</pre>
+{% endfor %}
+<hr/>
+{% endfor %}
+'''
+{% endraw %}
+
+html_message_formatter = reporters.MessageFormatter(
+    template=email_template,
+    template_type='html',
+    wantProperties=True,
+    wantSteps=True,
+    wantLogs=True,
+    ctx=dict(statuses=util.Results),
+)
+
+mn = reporters.MailNotifier(
+    fromaddr='taskotron@fedoraproject.org',
+    sendToInterestedUsers=False,
+    subject="XXX %(projectName)s %(result)s on {{ deployment_type }} %(builder)s",
+    mode=('failing', 'exception', 'warnings'),
+    extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'],
+    relayhost="bastion.phx2.fedoraproject.org",
+    messageFormatter=html_message_formatter)
+
+c['services'].append(mn)
+
+## ExecDB's push-notifications
+
+sp = reporters.HttpStatusPush(
+    serverUrl="{{ execdb_statuspush }}",
+    wantProperties=True,
+    wantSteps=True,
+)
+c['services'].append(sp)
+
+
+####### PROJECT IDENTITY
+
+# the 'title' string will appear at the top of this buildbot installation's
+# home pages (linked to the 'titleURL').
+
+c['title'] = "Taskotron"
+c['titleURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/"
+
+# the 'buildbotURL' string should point to the location where the buildbot's
+# internal web server is visible. This typically uses the port number set in
+# the 'www' entry below, but with an externally-visible host name which the
+# buildbot cannot figure out without some help.
+
+c['buildbotURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/"
+
+
+####### WEB
+
+c['www'] = {
+    'port': 8010,
+    'plugins': {
+        'waterfall_view': {},
+        'console_view': {},
+        'grid_view': {},
+    },
+    'change_hook_dialects': {
+        'base': True,
+    },
+    'auth': util.UserPasswordAuth({
+{% if deployment_type == 'dev' %}
+        "{{ dev_buildbot_user }}": "{{ dev_buildbot_password }}",
+{% elif deployment_type == 'stg' %}
+        "{{ stg_buildbot_user }}": "{{ stg_buildbot_password }}",
+{% elif deployment_type == 'prod' %}
+        "{{ prod_buildbot_user }}": "{{ prod_buildbot_password }}",
+{% elif deployment_type == 'local' %}
+        "{{ local_buildbot_user }}": "{{ local_buildbot_password }}",
+{% endif %}
+    }),
+}
+
+
+####### DB URL
+
+c['db'] = {
+    # This specifies what database buildbot uses to store its state.
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+    'db_url' : "postgresql://{{ buildmaster_db_user }}:{{ buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}",
+{% elif deployment_type == 'local' %}
+    'db_url' : "postgresql://{{ local_buildmaster_db_user }}:{{ local_buildmaster_db_password }}@127.0.0.1/{{ buildmaster_db_name }}",
+{% endif %}
+}
+
+# TODO: REMOVE THIS
+c['db'] = {
+    'db_url' : "sqlite:///state.sqlite",
+}

roles/taskotron/buildmaster/tasks/main.yml

@@ -2,17 +2,20 @@
 - name: start httpd (provided in the apache role)
   service: name=httpd state=started
 
-- name: ensure packages required for buildmaster are installed (yum)
+- name: ensure packages required for buildmaster are installed
-  package: name={{ item }} state=present
+  dnf: name={{ item }} state=present
   with_items:
-    - buildbot
     - buildbot-master
-    - libsemanage-python
+    - buildbot-www
-    - python-psycopg2
+    - python2-libsemanage
+    - python3-psycopg2
     - policycoreutils-python-utils
-  when: ansible_distribution_major_version|int < 22
+    - python3-treq
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
 
-- name: ensure packages required for buildmaster are installed (dnf)
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
+- name: ensure packages required for buildmaster are installed
   dnf: name={{ item }} state=present
   with_items:
     - buildbot
@@ -20,17 +23,12 @@
     - libsemanage-python
     - python-psycopg2
     - policycoreutils-python-utils
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
-
+############
-- name: ensure additional packages required for CI buildmaster are installed
-  package: name={{ item }} state=present
-  when: ansible_distribution_major_version|int < 22 and (deployment_type == 'qa-prod' or deployment_type == 'qa-stg')
-  with_items:
-    - python-novaclient
 
 - name: ensure additional packages required for CI buildmaster are installed
   dnf: name={{ item }} state=present
-  when: ansible_distribution_major_version|int > 21 and (deployment_type == 'qa-prod' or deployment_type == 'qa-stg')
+  when: deployment_type in ['qa-prod', 'qa-stg']
   with_items:
     - python-novaclient
 
@@ -50,7 +48,7 @@
   when: deployment_type in ['qa-stg']
 
 - name: ensure correct fcontext for buildmaster home (new)
-  file: path="{{ buildmaster_home }}(/. *)?" setype=var_lib_t owner=buildmaster group=buildmaster mode=0775 state=directory
+  file: path="{{ buildmaster_home }}" setype=var_lib_t owner=buildmaster group=buildmaster mode=0775 state=directory
   when: deployment_type in ['dev', 'stg', 'prod']
 
 #- name: make sure the selinux fcontext is restored
@@ -65,6 +63,12 @@
   notify:
     - reload httpd
 
+- name: ensure postgresql service is running
+  systemd:
+    name: postgresql
+    state: started
+    enabled: true
+
 - name: ensure buildmaster database is created
   when: buildmaster_db_host != 'localhost'
   delegate_to: "{{ buildmaster_db_host }}"

roles/taskotron/buildmaster/templates/buildmaster.conf.j2

@@ -1,4 +1,13 @@
 <Location /{{ buildmaster_endpoint }}/ >
-  ProxyPass http://127.0.0.1:8010/
+  ProxyPass "http://127.0.0.1:8010/"
-  ProxyPassReverse http://127.0.0.1:8010/
+  ProxyPassReverse "http://127.0.0.1:8010/"
+
+  ProxyPreserveHost On
+</Location>
+
+<Location /{{ buildmaster_endpoint }}/ws >
+  ProxyPass "ws://127.0.0.1:8010/ws"
+  ProxyPassReverse "ws://127.0.0.1:8010/ws"
+
+  ProxyPreserveHost On
 </Location>

roles/taskotron/buildslave-configure/defaults/main.yml

@@ -1,2 +1,6 @@
 ---
 taskotron_admin_email: root@localhost.localdomain
+slaves:
+  - user: ""
+    home: ""
+    dir: ""

roles/taskotron/buildslave-configure/tasks/main.yml

@@ -7,7 +7,10 @@
 - name: generate slave config
   become: true
   become_user: "{{ slave_user }}"
-  template: src=buildbot.tac.j2 dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
+  template: src={{ item }} dest={{ slave_dir }}/buildbot.tac mode=0600 owner={{ slave_user }} group={{ slave_user }}
+  with_first_found:
+    - 'buildbot.tac.j2.{{ deployment_type }}'
+    - buildbot.tac.j2
   when: deployment_type in ['local', 'qa-stg']
 
 - name: generate slave info
@@ -37,7 +40,10 @@
   lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }}
 
 - name: generate buildslave service file
-  template: src=buildslave.service.j2 dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
+  template: src={{ item }} dest=/lib/systemd/system/buildslave.service owner=root group=root mode=0744
+  with_first_found:
+    - 'buildslave.service.j2.{{ deployment_type }}'
+    - buildslave.service.j2
   when: deployment_type in ['local', 'qa-stg']
   register: buildslave_service
 
@@ -49,20 +55,38 @@
   service: name=buildslave enabled=yes state={{ (buildslave_service.changed) | ternary('restarted','started') }}
   when: deployment_type in ['local', 'qa-stg']
 
+- name: create slave
+  become: true
+  become_user: '{{ item.user }}'
+  command: creates={{ item.dir }} buildbot-worker create-worker {{ item.dir }} {{ buildmaster }} buildslave passwd
+  with_items:
+    - '{{ slaves }}'
+  when: deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: create slave
   become: true
   become_user: '{{ item.user }}'
   command: creates={{ item.dir }} buildslave create-slave {{ item.dir }} {{ buildmaster }} buildslave passwd
   with_items:
     - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
-  when: deployment_type in ['dev', 'stg', 'prod']
+  when: deployment_type in ['stg', 'prod']
+############
+
+- name: figure out which buildbot.tac.j2 file to use
+  set_fact: buildbot_tac_j2_file={{ item | basename }}
+  # this unfortunately searches in files/
+  with_first_found:
+    - '../templates/buildbot.tac.j2.{{ deployment_type }}'
+    - '../templates/buildbot.tac.j2'
 
 - name: generate slave config
   become: true
   become_user: '{{ item.user }}'
-  template: src=buildbot.tac.j2 dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
+  template: src={{ buildbot_tac_j2_file }} dest={{ item.dir }}/buildbot.tac mode=0600 owner={{ item.user }} group={{ slaves_group }}
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: deployment_type in ['dev', 'stg', 'prod']
 
 - name: generate slave admin info
@@ -70,7 +94,7 @@
   become_user: '{{ item.user }}'
   template: src=admin.j2 dest={{ item.dir }}/info/admin mode=0644 owner={{ item.user }} group={{ slaves_group }}
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: deployment_type in ['dev', 'stg', 'prod']
 
 - name: generate slave host info
@@ -78,41 +102,44 @@
   become_user: '{{ item.user }}'
   template: src=host.j2 dest={{ item.dir }}/info/host mode=0644 owner={{ item.user }} group={{ slaves_group }}
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: deployment_type in ['dev', 'stg', 'prod']
 
 - name: create slave ssh directory
   file: path={{ item.home }}/.ssh owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
 
 - name: install slave ssh private key
   copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa owner={{ item.user }} group={{ slaves_group }} mode=0600
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: buildslave_private_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
 
 - name: install slave ssh public key
   copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ item.home }}/.ssh/id_rsa.pub owner={{ item.user }} group={{ slaves_group }} mode=0644
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
 
 - name: make sure master is in known_hosts
   lineinfile: dest={{ item.home }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ item.user }} group={{ slaves_group }}
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: buildslave_public_sshkey_file is defined and deployment_type in ['dev', 'stg', 'prod']
 
 - name: ensure correct fcontext for buildslave dir
   file: path={{ item.home }}/slave setype=var_lib_t owner={{ item.user }} group={{ slaves_group }} mode=0700 state=directory
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: deployment_type in ['dev', 'stg', 'prod']
 
 - name: generate buildslave service file
-  template: src=buildslave@.service.j2 dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0644
+  template: src={{ item }} dest=/lib/systemd/system/buildslave@.service owner=root group=root mode=0644
+  with_first_found:
+    - 'buildslave@.service.j2.{{ deployment_type }}'
+    - buildslave@.service.j2
   when: deployment_type in ['dev', 'stg', 'prod']
   register: buildslave_service
 
@@ -123,5 +150,5 @@
 - name: start and enable buildslave services
   service: name=buildslave@{{ item.user }} enabled=yes state={{ (buildslave_service.changed) | ternary('restarted','started') }}
   with_items:
-    - '{{ slaves|default([dict(user="", home="", dir="")]) }}'
+    - '{{ slaves }}'
   when: deployment_type in ['dev', 'stg', 'prod']

roles/taskotron/buildslave-configure/templates/buildbot.tac.j2.dev

@@ -0,0 +1,53 @@
+
+import os
+
+from buildbot_worker.bot import Worker
+from twisted.application import service
+
+{% if deployment_type in ['local', 'qa-stg'] %}
+basedir = '/home/buildslave/slave'
+{% endif %}
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+basedir = '{{ item.dir }}'
+{% endif %}
+rotateLength = 10000000
+maxRotatedFiles = 10
+
+# if this is a relocatable tac file, get the directory containing the TAC
+if basedir == '.':
+    import os.path
+    basedir = os.path.abspath(os.path.dirname(__file__))
+
+# note: this line is matched against to check that this is a worker
+# directory; do not edit it.
+application = service.Application('buildbot-worker')
+
+from twisted.python.logfile import LogFile
+from twisted.python.log import ILogObserver, FileLogObserver
+logfile = LogFile.fromFullPath(
+    os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+    maxRotatedFiles=maxRotatedFiles)
+application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+buildmaster_host = '{{ buildmaster }}'
+port = 9989
+{% if deployment_type in ['local', 'qa-stg', 'qa-prod'] %}
+workername = '{{ buildslave_name }}'
+passwd = '{{ buildslave_password }}'
+{% endif %}
+{% if deployment_type in ['dev', 'stg', 'prod'] %}
+workername = '{{ item.user }}'
+passwd = '{{ buildslave_password }}'
+{% endif %}
+keepalive = 600
+umask = 0o0022
+maxdelay = 300
+numcpus = None
+allow_shutdown = None
+maxretries = None
+
+s = Worker(buildmaster_host, port, workername, passwd, basedir,
+           keepalive, umask=umask, maxdelay=maxdelay,
+           numcpus=numcpus, allow_shutdown=allow_shutdown,
+           maxRetries=maxretries)
+s.setServiceParent(application)

roles/taskotron/buildslave-configure/templates/buildslave.service.j2.dev

@@ -0,0 +1,15 @@
+[Unit]
+Description=Buildslave for taskbot
+After=network.target
+
+[Service]
+Type=forking
+# disabled because of https://pagure.io/taskotron/issue/236
+#PIDFile=/home/buildslave/slave/twistd.pid
+ExecStart=/usr/bin/buildbot-worker start /home/buildslave/slave/
+ExecStop=/usr/bin/buildbot-worker stop /home/buildslave/slave/
+User=buildslave
+Group=buildslave
+
+[Install]
+WantedBy=multi-user.target

roles/taskotron/buildslave-configure/templates/buildslave@.service.j2.dev

@@ -0,0 +1,15 @@
+[Unit]
+Description=Buildslave for taskotron
+After=network.target
+
+[Service]
+Type=forking
+# disabled because of https://pagure.io/taskotron/issue/236
+#PIDFile=/home/%i/slave/twistd.pid
+ExecStart=/usr/bin/buildbot-worker start /srv/buildslaves/%i/slave/
+ExecStop=/usr/bin/buildbot-worker stop /srv/buildslaves/%i/slave/
+User=%i
+Group={{ slaves_group }}
+
+[Install]
+WantedBy=multi-user.target

roles/taskotron/buildslave/tasks/main.yml

@@ -1,9 +1,33 @@
 ---
-- name: ensure packages required for buildslave are installed (dnf)
+- name: ensure packages required for buildslave are installed
+  dnf: name={{ item }} state=present
+  with_items:
+    - buildbot-worker
+    - git
+    - policycoreutils-python-utils
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
+- name: ensure packages required for buildslave are installed
   dnf: name={{ item }} state=present
   with_items:
     - buildbot-slave
     - git
+    - policycoreutils-python-utils
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
+############
+
+- name: ensure packages required for CI buildslave are installed
+  dnf: name={{ item }} state=present
+  with_items:
+    - testcloud
+    - mash
+    - mock
+    - koji
+    - python-fedora
+    - python-doit
+    - python2-rpmfluff
     - rpmlint
     - koji
     - bodhi-client
@@ -14,20 +38,7 @@
     - pylint
     - python-pep8
     - moreutils
-    - policycoreutils-python-utils
+  when: (deployment_type in ['qa-stg']) and (ansible_cmdline.ostree is not defined)
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
-
-- name: ensure packages required for CI buildslave are installed (dnf)
-  dnf: name={{ item }} state=present
-  with_items:
-    - testcloud
-    - mash
-    - mock
-    - koji
-    - python-fedora
-    - python-doit
-    - python2-rpmfluff
-  when: (deployment_type == 'qa-stg') and (ansible_distribution_major_version|int > 21) and (ansible_cmdline.ostree is not defined)
 
 - name: set the selinux fcontext type for the buildmaster_home to var_lib_t
   command: semanage fcontext -a -t var_lib_t "{{ slave_dir }}(/.*)?"
@@ -52,6 +63,13 @@
   group: name={{ slaves_group }}
   when: slaves_group is defined and (deployment_type in ['dev', 'stg', 'prod'])
 
+- name: create buildslaves home
+  file:
+    path: "{{ buildslaves_home }}"
+    state: directory
+    mode: 0755
+  when: slaves is defined and (deployment_type in ['dev', 'stg', 'prod'])
+
 - name: add buildslave users
   user: name={{ item.user }} group={{ slaves_group }} groups=testcloud,taskotron home={{ item.home }}
   with_items: "{{ slaves }}"
@@ -63,13 +81,13 @@
   when: slaves is defined and deployment_type in ['qa-prod', 'qa-stg']
 
 - name: make sure the selinux fcontext is restored
-  command: restorecon -R "{{ item.dir }}"
+  command: restorecon -R "{{ item.home }}"
   with_items: "{{ slaves }}"
   when: slaves is defined and deployment_type in ['dev', 'stg', 'prod', 'qa-prod', 'qa-stg']
 
 - name: disable ansible host key checking
   lineinfile:
     dest: /etc/ansible/ansible.cfg
-    regexp: '^#host_key_checking = False$'
+    regexp: '^#?host_key_checking = False$'
     line: 'host_key_checking = False'
   when: deployment_type in ['dev']

roles/taskotron/execdb/tasks/main.yml

@@ -2,15 +2,17 @@
 - name: start httpd (provided in the apache role)
   service: name=httpd state=started
 
-- name: ensure packages required for execdb are installed (yum)
+- name: ensure packages required for execdb are installed (dnf)
-  package: name={{ item }} state=present
+  dnf: name={{ item }} state=present
   with_items:
     - execdb
-    - mod_wsgi
+    - python3-mod_wsgi
-    - python-psycopg2
+    - python3-psycopg2
-    - libsemanage-python
+    - python2-libsemanage
-  when: ansible_distribution_major_version|int < 22
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
 
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: ensure packages required for execdb are installed (dnf)
   dnf: name={{ item }} state=present
   with_items:
@@ -18,7 +20,8 @@
     - mod_wsgi
     - python-psycopg2
     - libsemanage-python
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
+###########
 
 - name: ensure database is created
   delegate_to: "{{ execdb_db_host_machine }}"

roles/taskotron/resultsdb-backend/tasks/main.yml

@@ -2,23 +2,26 @@
 - name: start httpd (provided in the apache role)
   service: name=httpd state=started
 
-- name: ensure packages required for resultsdb are installed (yum)
-  package: name={{ item }} state=present
-  when: ansible_distribution_major_version|int < 22
-  with_items:
-    - resultsdb
-    - mod_wsgi
-    - python-psycopg2
-    - libsemanage-python
-
 - name: ensure packages required for resultsdb are installed (dnf)
   dnf: name={{ item }} state=present
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  with_items:
+    - resultsdb
+    - python3-mod_wsgi
+    - python3-psycopg2
+    - python2-libsemanage
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
+- name: ensure packages required for resultsdb are installed (dnf)
+  dnf: name={{ item }} state=present
   with_items:
     - resultsdb
     - mod_wsgi
     - python-psycopg2
     - libsemanage-python
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
+###########
 
 - name: ensure database is created
   delegate_to: "{{ resultsdb_db_host_machine }}"

roles/taskotron/resultsdb-frontend/tasks/main.yml

@@ -2,19 +2,22 @@
 - name: start httpd (provided in the apache role)
   service: name=httpd state=started
 
-- name: ensure packages required for resultsdb_frontend are installed (yum)
+- name: ensure packages required for resultsdb_frontend are installed (dnf)
-  package: name={{ item }} state=present
+  dnf: name={{ item }} state=present
-  when: ansible_distribution_major_version|int < 22
+  with_items:
-  with_items:
+    - resultsdb_frontend
-    - resultsdb_frontend
+    - python3-mod_wsgi
-    - mod_wsgi
+  when: ansible_cmdline.ostree is not defined and and deployment_type in ['dev']
-
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: ensure packages required for resultsdb_frontend are installed (dnf)
   dnf: name={{ item }} state=present
   with_items:
     - resultsdb_frontend
     - mod_wsgi
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
+###########
 
 - name: ensure selinux lets httpd talk to the network
   seboolean: name=httpd_can_network_connect persistent=yes state=yes

roles/taskotron/taskotron-client/tasks/main.yml

@@ -1,12 +1,12 @@
 ---
-- name: ensure packages required for libtaskotron are installed (yum)
+- name: ensure packages required for libtaskotron are installed
-  package: name={{ item }} state=present
+  dnf: name={{ item }} state=present
   with_items:
     - PyYAML
     - libtaskotron
   when: deployment_type == 'local'
 
-- name: ensure packages required for libtaskotron are installed (dnf)
+- name: ensure packages required for libtaskotron are installed
   dnf: name={{ item }} state=present
   with_items:
     - PyYAML
@@ -15,21 +15,13 @@
     - selinux-policy-devel
   when: deployment_type in ['dev', 'stg', 'prod']
 
-- name: ensure packages required for taskotron tasks are installed (yum)
+- name: ensure packages required for taskotron tasks are installed
-  package: name={{ item }} state=present
-  with_items:
-    - python-solv
-    - python-librepo
-    - rpmlint
-  when: ansible_distribution_major_version|int < 22
-
-- name: ensure packages required for taskotron tasks are installed (dnf)
   dnf: name={{ item }} state=present
   with_items:
     - python-solv
     - python-librepo
     - rpmlint
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined
 
 - name: generate taskotron.yaml config file
   template: src={{ item }} dest=/etc/taskotron/taskotron.yaml owner=root group=root mode=0644
@@ -109,16 +101,11 @@
   command: chdir=/root/ semodule -i /root/service-virt-transition.pp
   when: deployment_type in ['dev', 'stg', 'prod']
 
-- name: enable libvirtd
-  service: name=libvirtd state=started enabled=yes
-  when: ansible_distribution_major_version|int < 24
-
 - name: start and enable libvirt services
   service: name={{ item }} state=started enabled=yes
   with_items:
     - libvirtd
     - virtlogd
-  when: ansible_distribution_major_version|int >= 24
 
 - name: copy custom libvirt network config
   copy: src=default.xml dest=/etc/libvirt/qemu/networks/default.xml owner=root group=root mode=0600

roles/taskotron/taskotron-trigger/tasks/main.yml

@@ -1,24 +1,25 @@
 ---
-- name: ensure packages required for taskotron-trigger are installed (yum)
+- name: ensure packages required for taskotron-trigger are installed
-  package: name={{ item }} state=present
-  with_items:
-    - fedmsg
-    - fedmsg-hub
-    - taskotron-trigger
-    # needed for cron script
-    - python-resultsdb_api
-  when: ansible_distribution_major_version|int < 22
-
-- name: ensure packages required for taskotron-trigger are installed (dnf)
   dnf: name={{ item }} state=present
   with_items:
     - fedmsg
     - fedmsg-hub
     - taskotron-trigger
-    # needed for cron script
+    - python2-resultsdb_api  # TODO: remove when transition to buildbot 1.7 is complete
-    - python2-resultsdb_api
+    - python3-resultsdb_api
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined
 
+- name: generate trigger configuration
+  template: src={{ item }} dest=/etc/taskotron/trigger.cfg owner=root group=root mode=0744
+  with_first_found:
+    - trigger.cfg.j2.{{ deployment_type }}
+    - trigger.cfg.j2
+  notify:
+    - restart fedmsg-hub-3
+  when: deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: generate trigger configuration
   template: src={{ item }} dest=/etc/taskotron/trigger.cfg owner=root group=root mode=0744
   with_first_found:
@@ -26,7 +27,20 @@
     - trigger.cfg.j2
   notify:
     - restart fedmsg-hub
+  when: deployment_type in ['stg', 'prod']
+###########
 
+- name: generate trigger rules
+  template: src={{ item }} dest=/etc/taskotron/trigger_rules.yml owner=root group=root mode=0744
+  with_first_found:
+    - trigger_rules.yml.j2.{{ deployment_type }}
+    - trigger_rules.yml.j2
+  notify:
+    - restart fedmsg-hub-3
+  when: deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: generate trigger rules
   template: src={{ item }} dest=/etc/taskotron/trigger_rules.yml owner=root group=root mode=0744
   with_first_found:
@@ -34,11 +48,23 @@
     - trigger_rules.yml.j2
   notify:
     - restart fedmsg-hub
+  when: deployment_type in ['stg', 'prod']
+#########
 
+- name: copy fedmsg logging configuration
+  copy: src=logging.py dest=/etc/fedmsg.d/logging.py owner=root group=root mode=0744
+  notify:
+    - restart fedmsg-hub-3
+  when: deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: copy fedmsg logging configuration
   copy: src=logging.py dest=/etc/fedmsg.d/logging.py owner=root group=root mode=0744
   notify:
     - restart fedmsg-hub
+  when: deployment_type in ['stg', 'prod']
+###########
 
 - name: install fetch_activity
   copy: src=fetch_activity.py dest=/root/fetch_activity.py owner=root group=root mode=0644
@@ -54,5 +80,13 @@
 - name: copy tmpfiles.d for trigger distgit cache
   template: src=tmpfiles.d.trigger.conf.j2 dest=/etc/tmpfiles.d/trigger.conf owner=root group=root mode=0644
 
+- name: start and enable fedmsg-hub-3 service
+  service: name=fedmsg-hub-3 enabled=yes state=started
+  when: deployment_type in ['dev']
+
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: start and enable fedmsg-hub service
   service: name=fedmsg-hub enabled=yes state=started
+  when: deployment_type in ['stg', 'prod']
+###########

roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2

@@ -21,4 +21,3 @@ deployment_type = stg
 [koji_build_completed]
 ; critpath_filepath = {{ trigger_critpath_file }}
 critpath_filepath =
-

roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2.dev

@@ -0,0 +1,23 @@
+[buildbot]
+url = http://127.0.0.1:8010/change_hook
+
+[trigger]
+valid_arches = x86_64,armhfp
+koji_url = http://koji.fedoraproject.org/kojihub
+datagrepper_url = https://apps.fedoraproject.org/datagrepper/raw
+execdb_server = {{ execdb_server }}
+runner_type = BuildbotRunner
+job_logging = True
+joblog_file = {{ trigger_joblog_file }}
+fuse_delay = 900 ; 15 minutes in seconds
+git_cache_dir = {{ trigger_cache_dir }}
+rules_template = {{ trigger_rules_template_path }}
+{# enable this if you want taskotron-stg to react to just stg fedmsgs
+{% if deployment_type in ['stg'] %}
+deployment_type = stg
+{% endif%}
+#}
+
+[koji_build_completed]
+; critpath_filepath = {{ trigger_critpath_file }}
+critpath_filepath =

roles/taskotron/vault/tasks/main.yml

@@ -2,15 +2,17 @@
 - name: start httpd (provided in the apache role)
   service: name=httpd state=started
 
-- name: ensure packages required for vault are installed (yum)
+- name: ensure packages required for vault are installed (dnf)
-  package: name={{ item }} state=present
+  dnf: name={{ item }} state=present
   with_items:
     - vault
-    - mod_wsgi
+    - python3-mod_wsgi
-    - python-psycopg2
+    - python3-psycopg2
-    - libsemanage-python
+    - python2-libsemanage
-  when: ansible_distribution_major_version|int < 22
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['dev']
 
+###########
+## TODO: remove when transition to buildbot 1.7 is complete
 - name: ensure packages required for vault are installed (dnf)
   dnf: name={{ item }} state=present
   with_items:
@@ -18,7 +20,8 @@
     - mod_wsgi
     - python-psycopg2
     - libsemanage-python
-  when: ansible_distribution_major_version|int > 21 and ansible_cmdline.ostree is not defined
+  when: ansible_cmdline.ostree is not defined and deployment_type in ['stg', 'prod']
+###########
 
 - name: ensure database is created
   delegate_to: "{{ vault_db_host_machine }}"