From 8e562f0e9f7cb866ff7376a9ae532a01c2ffda3b Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 13 Aug 2014 11:47:41 +0000 Subject: [PATCH] Add qadevel-stg playbook --- inventory/group_vars/qadevel-stg | 24 ++ .../qadevel-stg.qa.fedoraproject.org | 17 + playbooks/groups/qadevel-stg.yml | 72 +++++ .../templates/ci.master.cfg.j2 | 303 ++++++++++++++++++ 4 files changed, 416 insertions(+) create mode 100644 playbooks/groups/qadevel-stg.yml create mode 100644 roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index 9992ec58be..3693728050 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -18,3 +18,27 @@ virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ m gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 hostname={{ inventory_hostname }}" --network=bridge=br0 --autostart --noautoconsole + +grokmirror_basedir: /var/lib/git/mirror +grokmirror_user: grokmirror +grokmirror_repos: + - { name: fedoraqa/taskotron-trigger, url: 'https://bitbucket.org/fedoraqa/taskotron-triger.git'} + - { name: fedoraqa/libtaskotron, url: 'https://bitbucket.org/fedoraqa/libtaskotron.git'} + - { name: fedoraqa/resultsdb, url: 'https://bitbucket.org/fedoraqa/resultsdb.git'} + - { name: fedoraqa/resultsdb_api, url: 'https://bitbucket.org/fedoraqa/resultsdb_api.git'} + - { name: fedoraqa/fake_fedorainfra, url: 'https://bitbucket.org/fedoraqa/fake_fedorainfra.git'} +buildmaster_db_host: db-qa01.qa.fedoraproject.org +buildmaster_db_name: buildmaster_ci_stg +buildmaster_template: ci.master.cfg.j2 +buildmaster_endpoint: taskmaster +buildslave_ssh_pubkey: '' +buildslave_port: 9989 +cgit_root_title: "Taskotron CI Stg Server Git Mirror" +buildmaster_dir: /home/buildmaster/master +buildslave_dir: /home/buildslave/slave +buildslave_poll_interval: 1800 +master_dir: /home/buildmaster/master +master_user: buildmaster +external_hostname: qadevel-stg.qa.fedoraproject.org +deployment_type: stg +tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] diff --git a/inventory/host_vars/qadevel-stg.qa.fedoraproject.org b/inventory/host_vars/qadevel-stg.qa.fedoraproject.org index 1096be1889..fe4ad6c3c9 100644 --- a/inventory/host_vars/qadevel-stg.qa.fedoraproject.org +++ b/inventory/host_vars/qadevel-stg.qa.fedoraproject.org @@ -8,3 +8,20 @@ volgroup: /dev/Guests00 eth0_ip: 10.5.124.181 vmhost: virthost-comm01.qa.fedoraproject.org datacenter: phx2 +fas_client_groups: sysadmin-qa,sysadmin-main + +# default virt install command is for a single nic-device +# define in another group file for more nics (see buildvm) +virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ mem_size }} + --disk bus=virtio,path={{ volgroup }}/{{ inventory_hostname }} + --vcpus={{ num_cpus }} -l {{ ks_repo }} -x + "ks={{ ks_url }} ip={{ eth0_ip }} netmask={{ nm }} + gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 + hostname={{ inventory_hostname }}" + --network=bridge=br0 --autostart --noautoconsole + +public_hostname: qadevel-stg.qa.fedoraproject.org +buildmaster: 10.5.124.181 + +buildslaves: + - qadevel-stg diff --git a/playbooks/groups/qadevel-stg.yml b/playbooks/groups/qadevel-stg.yml new file mode 100644 index 0000000000..127c5c5fb5 --- /dev/null +++ b/playbooks/groups/qadevel-stg.yml @@ -0,0 +1,72 @@ +--- +# create a new taskotron CI stg server +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars + +- name: make taskotron-ci staging + hosts: qadevel-stg + user: root + gather_facts: False + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: qadevel-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: base, tags:['base'] } + - { role: rkhunter, tags:['rkhunter'] } + - { role: nagios_client, tags:['nagios_client'] } + - hosts + - { role: fas_client, tags:['fas_client'] } + - { role: collectd/base, tags:['collectd_base'] } + - { role: yum-cron, tags:['yumcron'] } + - { role: sudo, tags:['sudo'] } + + tasks: + # this is how you include other task lists + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/apache.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: configure taskotron-ci master + hosts: qadevel-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: taskotron/grokmirror, tags: ['grokmirror'] } +# - { role: taskotron/cgit, tags: ['cgit'] } + - { role: taskotron/buildmaster, tags: ['buildmaster'] } + - { role: taskotron/buildmaster-configure, tags: ['buildmasterconfig'] } + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 new file mode 100644 index 0000000000..92ae6f7d7f --- /dev/null +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -0,0 +1,303 @@ +# -*- python -*- +# ex: set syntax=python: + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### BUILDSLAVES + +# The 'slaves' list defines the set of recognized buildslaves. Each element is +# a BuildSlave object, specifying a unique slave name and password. The same +# slave name and password must be configured on the slave. +from buildbot.buildslave import BuildSlave +c['slaves'] = [ + {% for buildslave in buildslaves %} + {% if deployment_type == 'stg' %} + BuildSlave("{{ buildslave }}", "{{ stg_buildslave_password }}"), + {% endif %} + {% if deployment_type == 'prod' %} + BuildSlave("{{ buildslave }}", "{{ prod_buildslave_password }}"), + {% endif %} + {% endfor %} + ] + +# 'slavePortnum' defines the TCP port to listen on for connections from slaves. +# This must match the value configured into the buildslaves (with their +# --master option) +c['slavePortnum'] = {{ buildslave_port }} + + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot clone of pyflakes. + +from buildbot.changes.gitpoller import GitPoller + +interval = {{ buildslave_poll_interval }} + +c['change_source'] = [] +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/taskotron-trigger/', + workdir='gitpoller-workdir-trigger', branch='develop', + pollinterval=interval, + project='trigger')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/libtaskotron/', + workdir='gitpoller-workdir-libtaskotron', branch='develop', + pollinterval=interval, + project='libtaskotron')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb/', + workdir='gitpoller-workdir-resultsdb', branch='develop', + pollinterval=interval, + project='resultsdb')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb_api/', + workdir='gitpoller-workdir-resultsdb_api', branch='develop', + pollinterval=interval, + project='resultsdb_api')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/fake_fedorainfra/', + workdir='gitpoller-workdir-fake_fedorainfra', branch='develop', + pollinterval=interval, + project='fake_fedorainfra')) + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.changes import filter +c['schedulers'] = [] + +c['schedulers'].append(SingleBranchScheduler( + name="trigger-scheduler", + change_filter=filter.ChangeFilter(project='trigger', branch='develop'), + treeStableTimer=None, + builderNames=["trigger-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="libtaskotron-scheduler", + change_filter=filter.ChangeFilter(project='libtaskotron', branch='develop'), + treeStableTimer=None, + builderNames=["libtaskotron-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="resultsdb-scheduler", + change_filter=filter.ChangeFilter(project='resultsdb', branch='develop'), + treeStableTimer=None, + builderNames=["resultsdb-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="fake_fedorainfra-scheduler", + change_filter=filter.ChangeFilter(project='fake_fedorainfra', branch='develop'), + treeStableTimer=None, + builderNames=["fake_fedorainfra-builder"])) + +c['schedulers'].append(ForceScheduler( + name="force", + builderNames=["trigger-builder", "libtaskotron-builder", "resultsdb-builder", "resultsdb_api-builder", "fake_fedorainfra-builder"])) + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which slaves can execute them. Note that any particular build will +# only take place on one slave. + +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.git import Git +from buildbot.steps.shell import ShellCommand + + +def create_test_factory(repo_name, func=True): + factory = BuildFactory() + factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%s/' % repo_name), + mode='full', method='clobber')) + factory.addStep(ShellCommand(command=['virtualenv', '--system-site-packages', 'env'])) + factory.addStep(ShellCommand(command=['bash', '-c', 'source env/bin/activate; pip install -r requirements.txt'])) + factory.addStep(ShellCommand(command=['bash', '-c', 'source env/bin/activate; TEST="true" py.test %s testing/' % '-F' if func else ''], name=repo_name)) + + return factory + + +trigger_factory = create_test_factory('taskotron-trigger') +libtaskotron_factory = create_test_factory('libtaskotron') +resultsdb_factory = create_test_factory('resultsdb') +resultsdb_api_factory = create_test_factory('resultsdb_api') +fake_fedorainfra_factory = create_test_factory('fake_fedorainfra') + + +from buildbot.config import BuilderConfig + +c['builders'] = [] +c['builders'].append( + BuilderConfig(name="trigger-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=trigger_factory)) +c['builders'].append( + BuilderConfig(name="libtaskotron-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=libtaskotron_factory)) +c['builders'].append( + BuilderConfig(name="resultsdb-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=resultsdb_factory)) +c['builders'].append( + BuilderConfig(name="resultsdb_api-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=resultsdb_api_factory)) +c['builders'].append( + BuilderConfig(name="fake_fedorainfra-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=fake_fedorainfra_factory)) + + +####### STATUS TARGETS + +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# including web pages, email senders, and IRC bots. +# +c['status'] = [] + +from buildbot.status import html +from buildbot.status.web import authz, auth + +authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + {% if deployment_type == 'stg' %} + auth=auth.BasicAuth([("{{ stg_buildbot_user }}","{{ stg_buildbot_password }}")]), + {% endif %} + {% if deployment_type == 'prod' %} + auth=auth.BasicAuth([("{{ prod_buildbot_user }}","{{ prod_buildbot_password }}")]), + {% endif %} + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, +) +c['status'].append(html.WebStatus(http_port=8080, authz=authz_cfg, + change_hook_dialects={'base':True})) +c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + + +from buildbot.status.mail import MailNotifier +from buildbot.status.builder import Results + +import cgi + +# http://docs.buildbot.net/current/manual/cfg-statustargets.html#mailnotifier +def html_message_formatter(mode, name, build, results, master_status): + """Provide a customized message to Buildbot's MailNotifier. + + The last 80 lines of the log are provided as well as the changes + relevant to the build. Message content is formatted as html. + """ + result = Results[results] + + limit_lines = 80 + text = list() + text.append(u'

Build status: %s

' % result.upper()) + text.append(u'') + text.append(u"" % build.getSlavename()) + if master_status.getURLForThing(build): + text.append(u'' + % (master_status.getURLForThing(build), + master_status.getURLForThing(build)) + ) + text.append(u'' % build.getReason()) + text.append(u'
Buildslave for this Build:%s
Complete logs for all build steps:%s
Build Reason:%s
') + text.append(u'
') + + logs = [] + steps = build.getSteps() + for step in steps: + if step.getName().endswith('test'): + logs = step.getLogs() + break + + # logs within a step are in reverse order. Search back until we find stdio + for log in reversed(logs): + if log.getName() == 'stdio': + break + + name = "%s.%s" % (log.getStep().getName(), log.getName()) + content = log.getText().splitlines() # Note: can be VERY LARGE + url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), + log.getStep().getName(), + log.getName()) + + text.append(u'Detailed log of last build step: %s' + % (url, url)) + text.append(u'
') + text.append(u'

Last %d lines of "%s"

' % (limit_lines, name)) + unilist = list() + for line in content[len(content)-limit_lines:]: + unilist.append(cgi.escape(unicode(line,'utf-8'))) + text.append(u'
')
+        text.extend(unilist)
+        text.append(u'
') + text.append(u'

') + text.append(u'-The Buildbot') + return { + 'body': u"\n".join(text), + 'type': 'html' + } +''' +mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', + sendToInterestedUsers=False, + subject="%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s", + mode=('failing', 'exception', 'warnings'), + extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], + relayhost="bastion.phx2.fedoraproject.org", + messageFormatter=html_message_formatter) + + +c['status'].append(mn) +''' + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot +# installation's html.WebStatus home page (linked to the +# 'titleURL') and is embedded in the title of the waterfall HTML page. + +c['title'] = "Taskotron CI" +c['titleURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server (usually the html.WebStatus page) is visible. This +# typically uses the port number set in the Waterfall 'status' entry, but +# with an externally-visible host name which the buildbot cannot figure out +# without some help. + +c['buildbotURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/" + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + #'db_url' : "sqlite:///state.sqlite", + {% if deployment_type == 'stg' %} + 'db_url' : "postgresql://{{ stg_buildmaster_db_user }}:{{ stg_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", + {% endif %} + {% if deployment_type == 'prod' %} + 'db_url' : "postgresql://{{ prod_buildmaster_db_user }}:{{ prod_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", + {% endif %} +}