From c744698e0cd9f901403016a15b3f612780f9468b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 12 Aug 2014 03:36:21 +0000 Subject: [PATCH 001/193] Tweak openvpn base and keys --- playbooks/groups/keyserver.yml | 3 +-- roles/openvpn/base/tasks/main.yml | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/playbooks/groups/keyserver.yml b/playbooks/groups/keyserver.yml index 2a7b403ecc..bc30c3a194 100644 --- a/playbooks/groups/keyserver.yml +++ b/playbooks/groups/keyserver.yml @@ -38,12 +38,11 @@ - nagios_client - hosts - fas_client - - fedmsg/base - - keyserver - sudo - collectd/base - { role: openvpn/client, when: env != "staging" } + - keyserver tasks: - include: "{{ tasks }}/yumrepos.yml" diff --git a/roles/openvpn/base/tasks/main.yml b/roles/openvpn/base/tasks/main.yml index 47a5c9a00a..62ebfd8b93 100644 --- a/roles/openvpn/base/tasks/main.yml +++ b/roles/openvpn/base/tasks/main.yml @@ -9,16 +9,9 @@ - packages - name: Install certificate and key - file: src={{ item.file }} - dest={{ item.dest }} - owner=root group=root mode={{ item.mode }} - with_items: - - { file: "{{ puppet_private }}/vpn/openvpn/keys/ca.crt", - dest: /etc/openvpn/ca.crt, - mode: '0600' } -# - { file: "{{ puppet_private }}/vpn/openvpn/keys/crl.pem", -# dest: /etc/openvpn/crl.pem, -# mode: '0644' } + file: src={{ puppet_private }}/vpn/openvpn/keys/ca.crt + dest=/etc/openvpn/ca.crt + owner=root group=root mode=0600 tags: - install - openvpn From b43f8e30b3cc43ce119f30aed20e88c207eb919b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 12 Aug 2014 03:41:16 +0000 Subject: [PATCH 002/193] Try copy --- roles/openvpn/base/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openvpn/base/tasks/main.yml b/roles/openvpn/base/tasks/main.yml index 62ebfd8b93..890ec5dbe8 100644 --- a/roles/openvpn/base/tasks/main.yml +++ b/roles/openvpn/base/tasks/main.yml @@ -9,7 +9,7 @@ - packages - name: Install certificate and key - file: src={{ puppet_private }}/vpn/openvpn/keys/ca.crt + copy: src={{ puppet_private }}/vpn/openvpn/keys/ca.crt dest=/etc/openvpn/ca.crt owner=root group=root mode=0600 tags: From b77b40ff2c15c1d0c0550c3931afc9149c09dfc5 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 12 Aug 2014 03:49:58 +0000 Subject: [PATCH 003/193] Try copy here too. --- roles/openvpn/client/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openvpn/client/tasks/main.yml b/roles/openvpn/client/tasks/main.yml index c29ce8e8da..74f9a38b71 100644 --- a/roles/openvpn/client/tasks/main.yml +++ b/roles/openvpn/client/tasks/main.yml @@ -10,7 +10,7 @@ - openvpn - name: Install configuration files - file: src={{ item.file }} + copy: src={{ item.file }} dest={{ item.dest }} owner=root group=root mode={{ item.mode }} with_items: From 7016126b11a1be47e5587f5d99313df169c8d9d6 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 12 Aug 2014 14:26:25 +0000 Subject: [PATCH 004/193] Adjust run fasclient playbook a bit. --- playbooks/run_fasClient.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/playbooks/run_fasClient.yml b/playbooks/run_fasClient.yml index abd138e2f1..66b56360cd 100644 --- a/playbooks/run_fasClient.yml +++ b/playbooks/run_fasClient.yml @@ -1,9 +1,12 @@ -# Run `fasClient` on all hosts, 3 hosts at a time +# Run `fasClient` on all hosts, N hosts at a time +# +# We exclude builders, persistent-cloud and jenkins-cloud as they don't have fasclient +# - name: run fasClient - hosts: all + hosts: all:!builders:!persistent-cloud:!jenkins-cloud:!bkernel user: root - serial: 15 + serial: 20 gather_facts: False tasks: From 6455819a83c2514877b0259dc3da850c1f40aac6 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 12 Aug 2014 16:54:28 +0000 Subject: [PATCH 005/193] Add a small delay here in vhost_reboot --- playbooks/vhost_reboot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/vhost_reboot.yml b/playbooks/vhost_reboot.yml index 7012421a44..b45c8fe234 100644 --- a/playbooks/vhost_reboot.yml +++ b/playbooks/vhost_reboot.yml @@ -83,7 +83,7 @@ local_action: wait_for host={{ target }} port=22 delay=120 timeout=900 search_regex=OpenSSH - name: wait for libvirtd to come back on the virthost - wait_for: path=/var/run/libvirtd.pid state=present + wait_for: path=/var/run/libvirtd.pid state=present delay=10 - name: look up vmlist virt: command=list_vms From 8e562f0e9f7cb866ff7376a9ae532a01c2ffda3b Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 13 Aug 2014 11:47:41 +0000 Subject: [PATCH 006/193] Add qadevel-stg playbook --- inventory/group_vars/qadevel-stg | 24 ++ .../qadevel-stg.qa.fedoraproject.org | 17 + playbooks/groups/qadevel-stg.yml | 72 +++++ .../templates/ci.master.cfg.j2 | 303 ++++++++++++++++++ 4 files changed, 416 insertions(+) create mode 100644 playbooks/groups/qadevel-stg.yml create mode 100644 roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index 9992ec58be..3693728050 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -18,3 +18,27 @@ virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ m gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 hostname={{ inventory_hostname }}" --network=bridge=br0 --autostart --noautoconsole + +grokmirror_basedir: /var/lib/git/mirror +grokmirror_user: grokmirror +grokmirror_repos: + - { name: fedoraqa/taskotron-trigger, url: 'https://bitbucket.org/fedoraqa/taskotron-triger.git'} + - { name: fedoraqa/libtaskotron, url: 'https://bitbucket.org/fedoraqa/libtaskotron.git'} + - { name: fedoraqa/resultsdb, url: 'https://bitbucket.org/fedoraqa/resultsdb.git'} + - { name: fedoraqa/resultsdb_api, url: 'https://bitbucket.org/fedoraqa/resultsdb_api.git'} + - { name: fedoraqa/fake_fedorainfra, url: 'https://bitbucket.org/fedoraqa/fake_fedorainfra.git'} +buildmaster_db_host: db-qa01.qa.fedoraproject.org +buildmaster_db_name: buildmaster_ci_stg +buildmaster_template: ci.master.cfg.j2 +buildmaster_endpoint: taskmaster +buildslave_ssh_pubkey: '' +buildslave_port: 9989 +cgit_root_title: "Taskotron CI Stg Server Git Mirror" +buildmaster_dir: /home/buildmaster/master +buildslave_dir: /home/buildslave/slave +buildslave_poll_interval: 1800 +master_dir: /home/buildmaster/master +master_user: buildmaster +external_hostname: qadevel-stg.qa.fedoraproject.org +deployment_type: stg +tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] diff --git a/inventory/host_vars/qadevel-stg.qa.fedoraproject.org b/inventory/host_vars/qadevel-stg.qa.fedoraproject.org index 1096be1889..fe4ad6c3c9 100644 --- a/inventory/host_vars/qadevel-stg.qa.fedoraproject.org +++ b/inventory/host_vars/qadevel-stg.qa.fedoraproject.org @@ -8,3 +8,20 @@ volgroup: /dev/Guests00 eth0_ip: 10.5.124.181 vmhost: virthost-comm01.qa.fedoraproject.org datacenter: phx2 +fas_client_groups: sysadmin-qa,sysadmin-main + +# default virt install command is for a single nic-device +# define in another group file for more nics (see buildvm) +virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ mem_size }} + --disk bus=virtio,path={{ volgroup }}/{{ inventory_hostname }} + --vcpus={{ num_cpus }} -l {{ ks_repo }} -x + "ks={{ ks_url }} ip={{ eth0_ip }} netmask={{ nm }} + gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 + hostname={{ inventory_hostname }}" + --network=bridge=br0 --autostart --noautoconsole + +public_hostname: qadevel-stg.qa.fedoraproject.org +buildmaster: 10.5.124.181 + +buildslaves: + - qadevel-stg diff --git a/playbooks/groups/qadevel-stg.yml b/playbooks/groups/qadevel-stg.yml new file mode 100644 index 0000000000..127c5c5fb5 --- /dev/null +++ b/playbooks/groups/qadevel-stg.yml @@ -0,0 +1,72 @@ +--- +# create a new taskotron CI stg server +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars + +- name: make taskotron-ci staging + hosts: qadevel-stg + user: root + gather_facts: False + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: qadevel-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: base, tags:['base'] } + - { role: rkhunter, tags:['rkhunter'] } + - { role: nagios_client, tags:['nagios_client'] } + - hosts + - { role: fas_client, tags:['fas_client'] } + - { role: collectd/base, tags:['collectd_base'] } + - { role: yum-cron, tags:['yumcron'] } + - { role: sudo, tags:['sudo'] } + + tasks: + # this is how you include other task lists + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/apache.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: configure taskotron-ci master + hosts: qadevel-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: taskotron/grokmirror, tags: ['grokmirror'] } +# - { role: taskotron/cgit, tags: ['cgit'] } + - { role: taskotron/buildmaster, tags: ['buildmaster'] } + - { role: taskotron/buildmaster-configure, tags: ['buildmasterconfig'] } + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 new file mode 100644 index 0000000000..92ae6f7d7f --- /dev/null +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -0,0 +1,303 @@ +# -*- python -*- +# ex: set syntax=python: + +# This is a sample buildmaster config file. It must be installed as +# 'master.cfg' in your buildmaster's base directory. + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +####### BUILDSLAVES + +# The 'slaves' list defines the set of recognized buildslaves. Each element is +# a BuildSlave object, specifying a unique slave name and password. The same +# slave name and password must be configured on the slave. +from buildbot.buildslave import BuildSlave +c['slaves'] = [ + {% for buildslave in buildslaves %} + {% if deployment_type == 'stg' %} + BuildSlave("{{ buildslave }}", "{{ stg_buildslave_password }}"), + {% endif %} + {% if deployment_type == 'prod' %} + BuildSlave("{{ buildslave }}", "{{ prod_buildslave_password }}"), + {% endif %} + {% endfor %} + ] + +# 'slavePortnum' defines the TCP port to listen on for connections from slaves. +# This must match the value configured into the buildslaves (with their +# --master option) +c['slavePortnum'] = {{ buildslave_port }} + + +####### CHANGESOURCES + +# the 'change_source' setting tells the buildmaster how it should find out +# about source code changes. Here we point to the buildbot clone of pyflakes. + +from buildbot.changes.gitpoller import GitPoller + +interval = {{ buildslave_poll_interval }} + +c['change_source'] = [] +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/taskotron-trigger/', + workdir='gitpoller-workdir-trigger', branch='develop', + pollinterval=interval, + project='trigger')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/libtaskotron/', + workdir='gitpoller-workdir-libtaskotron', branch='develop', + pollinterval=interval, + project='libtaskotron')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb/', + workdir='gitpoller-workdir-resultsdb', branch='develop', + pollinterval=interval, + project='resultsdb')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb_api/', + workdir='gitpoller-workdir-resultsdb_api', branch='develop', + pollinterval=interval, + project='resultsdb_api')) + +c['change_source'].append(GitPoller( + '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/fake_fedorainfra/', + workdir='gitpoller-workdir-fake_fedorainfra', branch='develop', + pollinterval=interval, + project='fake_fedorainfra')) + +####### SCHEDULERS + +# Configure the Schedulers, which decide how to react to incoming changes. In this +# case, just kick off a 'runtests' build + +from buildbot.schedulers.basic import SingleBranchScheduler +from buildbot.schedulers.forcesched import ForceScheduler +from buildbot.changes import filter +c['schedulers'] = [] + +c['schedulers'].append(SingleBranchScheduler( + name="trigger-scheduler", + change_filter=filter.ChangeFilter(project='trigger', branch='develop'), + treeStableTimer=None, + builderNames=["trigger-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="libtaskotron-scheduler", + change_filter=filter.ChangeFilter(project='libtaskotron', branch='develop'), + treeStableTimer=None, + builderNames=["libtaskotron-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="resultsdb-scheduler", + change_filter=filter.ChangeFilter(project='resultsdb', branch='develop'), + treeStableTimer=None, + builderNames=["resultsdb-builder"])) + +c['schedulers'].append(SingleBranchScheduler( + name="fake_fedorainfra-scheduler", + change_filter=filter.ChangeFilter(project='fake_fedorainfra', branch='develop'), + treeStableTimer=None, + builderNames=["fake_fedorainfra-builder"])) + +c['schedulers'].append(ForceScheduler( + name="force", + builderNames=["trigger-builder", "libtaskotron-builder", "resultsdb-builder", "resultsdb_api-builder", "fake_fedorainfra-builder"])) + +####### BUILDERS + +# The 'builders' list defines the Builders, which tell Buildbot how to perform a build: +# what steps, and which slaves can execute them. Note that any particular build will +# only take place on one slave. + +from buildbot.process.factory import BuildFactory +from buildbot.steps.source.git import Git +from buildbot.steps.shell import ShellCommand + + +def create_test_factory(repo_name, func=True): + factory = BuildFactory() + factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%s/' % repo_name), + mode='full', method='clobber')) + factory.addStep(ShellCommand(command=['virtualenv', '--system-site-packages', 'env'])) + factory.addStep(ShellCommand(command=['bash', '-c', 'source env/bin/activate; pip install -r requirements.txt'])) + factory.addStep(ShellCommand(command=['bash', '-c', 'source env/bin/activate; TEST="true" py.test %s testing/' % '-F' if func else ''], name=repo_name)) + + return factory + + +trigger_factory = create_test_factory('taskotron-trigger') +libtaskotron_factory = create_test_factory('libtaskotron') +resultsdb_factory = create_test_factory('resultsdb') +resultsdb_api_factory = create_test_factory('resultsdb_api') +fake_fedorainfra_factory = create_test_factory('fake_fedorainfra') + + +from buildbot.config import BuilderConfig + +c['builders'] = [] +c['builders'].append( + BuilderConfig(name="trigger-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=trigger_factory)) +c['builders'].append( + BuilderConfig(name="libtaskotron-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=libtaskotron_factory)) +c['builders'].append( + BuilderConfig(name="resultsdb-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=resultsdb_factory)) +c['builders'].append( + BuilderConfig(name="resultsdb_api-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=resultsdb_api_factory)) +c['builders'].append( + BuilderConfig(name="fake_fedorainfra-builder", + slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], + factory=fake_fedorainfra_factory)) + + +####### STATUS TARGETS + +# 'status' is a list of Status Targets. The results of each build will be +# pushed to these targets. buildbot/status/*.py has a variety to choose from, +# including web pages, email senders, and IRC bots. +# +c['status'] = [] + +from buildbot.status import html +from buildbot.status.web import authz, auth + +authz_cfg=authz.Authz( + # change any of these to True to enable; see the manual for more + # options + {% if deployment_type == 'stg' %} + auth=auth.BasicAuth([("{{ stg_buildbot_user }}","{{ stg_buildbot_password }}")]), + {% endif %} + {% if deployment_type == 'prod' %} + auth=auth.BasicAuth([("{{ prod_buildbot_user }}","{{ prod_buildbot_password }}")]), + {% endif %} + gracefulShutdown = False, + forceBuild = 'auth', # use this to test your slave once it is set up + forceAllBuilds = False, + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, +) +c['status'].append(html.WebStatus(http_port=8080, authz=authz_cfg, + change_hook_dialects={'base':True})) +c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + + +from buildbot.status.mail import MailNotifier +from buildbot.status.builder import Results + +import cgi + +# http://docs.buildbot.net/current/manual/cfg-statustargets.html#mailnotifier +def html_message_formatter(mode, name, build, results, master_status): + """Provide a customized message to Buildbot's MailNotifier. + + The last 80 lines of the log are provided as well as the changes + relevant to the build. Message content is formatted as html. + """ + result = Results[results] + + limit_lines = 80 + text = list() + text.append(u'

Build status: %s

' % result.upper()) + text.append(u'') + text.append(u"" % build.getSlavename()) + if master_status.getURLForThing(build): + text.append(u'' + % (master_status.getURLForThing(build), + master_status.getURLForThing(build)) + ) + text.append(u'' % build.getReason()) + text.append(u'
Buildslave for this Build:%s
Complete logs for all build steps:%s
Build Reason:%s
') + text.append(u'
') + + logs = [] + steps = build.getSteps() + for step in steps: + if step.getName().endswith('test'): + logs = step.getLogs() + break + + # logs within a step are in reverse order. Search back until we find stdio + for log in reversed(logs): + if log.getName() == 'stdio': + break + + name = "%s.%s" % (log.getStep().getName(), log.getName()) + content = log.getText().splitlines() # Note: can be VERY LARGE + url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), + log.getStep().getName(), + log.getName()) + + text.append(u'Detailed log of last build step: %s' + % (url, url)) + text.append(u'
') + text.append(u'

Last %d lines of "%s"

' % (limit_lines, name)) + unilist = list() + for line in content[len(content)-limit_lines:]: + unilist.append(cgi.escape(unicode(line,'utf-8'))) + text.append(u'
')
+        text.extend(unilist)
+        text.append(u'
') + text.append(u'

') + text.append(u'-The Buildbot') + return { + 'body': u"\n".join(text), + 'type': 'html' + } +''' +mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', + sendToInterestedUsers=False, + subject="%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s", + mode=('failing', 'exception', 'warnings'), + extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], + relayhost="bastion.phx2.fedoraproject.org", + messageFormatter=html_message_formatter) + + +c['status'].append(mn) +''' + +####### PROJECT IDENTITY + +# the 'title' string will appear at the top of this buildbot +# installation's html.WebStatus home page (linked to the +# 'titleURL') and is embedded in the title of the waterfall HTML page. + +c['title'] = "Taskotron CI" +c['titleURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/" + +# the 'buildbotURL' string should point to the location where the buildbot's +# internal web server (usually the html.WebStatus page) is visible. This +# typically uses the port number set in the Waterfall 'status' entry, but +# with an externally-visible host name which the buildbot cannot figure out +# without some help. + +c['buildbotURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/" + +####### DB URL + +c['db'] = { + # This specifies what database buildbot uses to store its state. You can leave + # this at its default for all but the largest installations. + #'db_url' : "sqlite:///state.sqlite", + {% if deployment_type == 'stg' %} + 'db_url' : "postgresql://{{ stg_buildmaster_db_user }}:{{ stg_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", + {% endif %} + {% if deployment_type == 'prod' %} + 'db_url' : "postgresql://{{ prod_buildmaster_db_user }}:{{ prod_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", + {% endif %} +} From 3288718ac7b62c8d49534a2f93a415ddb5c284e4 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Aug 2014 14:13:53 +0000 Subject: [PATCH 007/193] Just use IPs for staging. moksha isn't doing the right thing with the hostnames. --- roles/bugzilla2fedmsg/templates/bugzilla2fedmsg.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/bugzilla2fedmsg/templates/bugzilla2fedmsg.ini b/roles/bugzilla2fedmsg/templates/bugzilla2fedmsg.ini index 83d8309e8e..b796e7c3f5 100644 --- a/roles/bugzilla2fedmsg/templates/bugzilla2fedmsg.ini +++ b/roles/bugzilla2fedmsg/templates/bugzilla2fedmsg.ini @@ -20,7 +20,8 @@ bugzilla.password = {{ bugzilla_password }} # Stomp broker configuration. {% if env == 'staging' %} -stomp_uri = fuse-fabric-01-stg.jboss.org:61617,fuse-fabric-02-stg.jboss.org:61617,fuse-fabric-03-stg.jboss.org:61617 +#stomp_uri = fuse-fabric-01-stg.jboss.org:61617,fuse-fabric-02-stg.jboss.org:61617,fuse-fabric-03-stg.jboss.org:61617 +stomp_uri = 10.34.40.176:61617,10.34.40.177:61617,10.34.40.178:61617 stomp_ssl_crt = /etc/pki/fedmsg/fedora.devel.engineering.redhat.com.crt stomp_ssl_key = /etc/pki/fedmsg/fedora.devel.engineering.redhat.com.key {% else %} From 969d20354365e60b6628362dfb8be648b696d118 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 13 Aug 2014 14:28:21 +0000 Subject: [PATCH 008/193] qadevel-stg: do not use grokmirror and postgresql --- inventory/group_vars/qadevel-stg | 10 ------- playbooks/groups/qadevel-stg.yml | 2 -- .../templates/ci.master.cfg.j2 | 28 ++++++++----------- 3 files changed, 11 insertions(+), 29 deletions(-) diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index 3693728050..256a69f943 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -19,21 +19,11 @@ virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ m hostname={{ inventory_hostname }}" --network=bridge=br0 --autostart --noautoconsole -grokmirror_basedir: /var/lib/git/mirror -grokmirror_user: grokmirror -grokmirror_repos: - - { name: fedoraqa/taskotron-trigger, url: 'https://bitbucket.org/fedoraqa/taskotron-triger.git'} - - { name: fedoraqa/libtaskotron, url: 'https://bitbucket.org/fedoraqa/libtaskotron.git'} - - { name: fedoraqa/resultsdb, url: 'https://bitbucket.org/fedoraqa/resultsdb.git'} - - { name: fedoraqa/resultsdb_api, url: 'https://bitbucket.org/fedoraqa/resultsdb_api.git'} - - { name: fedoraqa/fake_fedorainfra, url: 'https://bitbucket.org/fedoraqa/fake_fedorainfra.git'} -buildmaster_db_host: db-qa01.qa.fedoraproject.org buildmaster_db_name: buildmaster_ci_stg buildmaster_template: ci.master.cfg.j2 buildmaster_endpoint: taskmaster buildslave_ssh_pubkey: '' buildslave_port: 9989 -cgit_root_title: "Taskotron CI Stg Server Git Mirror" buildmaster_dir: /home/buildmaster/master buildslave_dir: /home/buildslave/slave buildslave_poll_interval: 1800 diff --git a/playbooks/groups/qadevel-stg.yml b/playbooks/groups/qadevel-stg.yml index 127c5c5fb5..f20beca3ca 100644 --- a/playbooks/groups/qadevel-stg.yml +++ b/playbooks/groups/qadevel-stg.yml @@ -63,8 +63,6 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: - - { role: taskotron/grokmirror, tags: ['grokmirror'] } -# - { role: taskotron/cgit, tags: ['cgit'] } - { role: taskotron/buildmaster, tags: ['buildmaster'] } - { role: taskotron/buildmaster-configure, tags: ['buildmasterconfig'] } diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 index 92ae6f7d7f..d1bf48e86e 100644 --- a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -17,10 +17,10 @@ from buildbot.buildslave import BuildSlave c['slaves'] = [ {% for buildslave in buildslaves %} {% if deployment_type == 'stg' %} - BuildSlave("{{ buildslave }}", "{{ stg_buildslave_password }}"), + BuildSlave("{{ buildslave }}", "{{ qadevel_stg_buildslave_password }}"), {% endif %} {% if deployment_type == 'prod' %} - BuildSlave("{{ buildslave }}", "{{ prod_buildslave_password }}"), + BuildSlave("{{ buildslave }}", "{{ qadevel_prod_buildslave_password }}"), {% endif %} {% endfor %} ] @@ -42,31 +42,31 @@ interval = {{ buildslave_poll_interval }} c['change_source'] = [] c['change_source'].append(GitPoller( - '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/taskotron-trigger/', + 'https://bitbucket.org/fedoraqa/taskotron-trigger.git', workdir='gitpoller-workdir-trigger', branch='develop', pollinterval=interval, project='trigger')) c['change_source'].append(GitPoller( - '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/libtaskotron/', + 'https://bitbucket.org/fedoraqa/libtaskotron.git', workdir='gitpoller-workdir-libtaskotron', branch='develop', pollinterval=interval, project='libtaskotron')) c['change_source'].append(GitPoller( - '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb/', + 'https://bitbucket.org/fedoraqa/resultsdb.git', workdir='gitpoller-workdir-resultsdb', branch='develop', pollinterval=interval, project='resultsdb')) c['change_source'].append(GitPoller( - '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/resultsdb_api/', + 'https://bitbucket.org/fedoraqa/resultsdb_api.git', workdir='gitpoller-workdir-resultsdb_api', branch='develop', pollinterval=interval, project='resultsdb_api')) c['change_source'].append(GitPoller( - '{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/fake_fedorainfra/', + 'https://bitbucket.org/fedoraqa/fake_fedorainfra.git', workdir='gitpoller-workdir-fake_fedorainfra', branch='develop', pollinterval=interval, project='fake_fedorainfra')) @@ -122,7 +122,7 @@ from buildbot.steps.shell import ShellCommand def create_test_factory(repo_name, func=True): factory = BuildFactory() - factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%s/' % repo_name), + factory.addStep(Git(repourl=Interpolate('https://bitbucket.org/fedoraqa/%s.git' % repo_name), mode='full', method='clobber')) factory.addStep(ShellCommand(command=['virtualenv', '--system-site-packages', 'env'])) factory.addStep(ShellCommand(command=['bash', '-c', 'source env/bin/activate; pip install -r requirements.txt'])) @@ -178,10 +178,10 @@ authz_cfg=authz.Authz( # change any of these to True to enable; see the manual for more # options {% if deployment_type == 'stg' %} - auth=auth.BasicAuth([("{{ stg_buildbot_user }}","{{ stg_buildbot_password }}")]), + auth=auth.BasicAuth([("{{ qadevel_stg_buildbot_master_user }}","{{ qadevel_stg_buildbot_master_password }}")]), {% endif %} {% if deployment_type == 'prod' %} - auth=auth.BasicAuth([("{{ prod_buildbot_user }}","{{ prod_buildbot_password }}")]), + auth=auth.BasicAuth([("{{ qadevel_prod_buildbot_master_user }}","{{ qadevel_prod_buildbot_master_password }}")]), {% endif %} gracefulShutdown = False, forceBuild = 'auth', # use this to test your slave once it is set up @@ -293,11 +293,5 @@ c['buildbotURL'] = "http://{{ external_hostname }}/{{buildmaster_endpoint}}/" c['db'] = { # This specifies what database buildbot uses to store its state. You can leave # this at its default for all but the largest installations. - #'db_url' : "sqlite:///state.sqlite", - {% if deployment_type == 'stg' %} - 'db_url' : "postgresql://{{ stg_buildmaster_db_user }}:{{ stg_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", - {% endif %} - {% if deployment_type == 'prod' %} - 'db_url' : "postgresql://{{ prod_buildmaster_db_user }}:{{ prod_buildmaster_db_password }}@{{ buildmaster_db_host }}/{{ buildmaster_db_name }}", - {% endif %} + 'db_url' : "sqlite:///state.sqlite", } From 87ab2a45cec095c41143372df67f250b2728bc52 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 13 Aug 2014 14:33:31 +0000 Subject: [PATCH 009/193] buildmaster: fix variable typo in mail subject --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 8882506269..a28a5b80c8 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(prop:taskname) %(result)s in %(projectName)s on %(builder)s", + subject="%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From 35640c4388086fbd074ee57dbf88e4f779a03060 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 13 Aug 2014 15:56:22 +0000 Subject: [PATCH 010/193] Some more small run fasClient playbook fixes. --- playbooks/run_fasClient.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/playbooks/run_fasClient.yml b/playbooks/run_fasClient.yml index 66b56360cd..06604167cf 100644 --- a/playbooks/run_fasClient.yml +++ b/playbooks/run_fasClient.yml @@ -3,16 +3,22 @@ # We exclude builders, persistent-cloud and jenkins-cloud as they don't have fasclient # -- name: run fasClient - hosts: all:!builders:!persistent-cloud:!jenkins-cloud:!bkernel +- name: run fasClient -i + hosts: all:!builders:!persistent-cloud:!jenkins-cloud:!bkernel:!*composer* user: root - serial: 20 + serial: 25 gather_facts: False tasks: - name: actually run fasClient -i command: fasClient -i +- name: run fasClient -a + hosts: bastion01.phx2.fedoraproject.org:bastion02.phx2.fedoraproject.org + user: root + gather_facts: False + + tasks: - name: actually run fasClient -a command: fasClient -a when: inventory_hostname_short.startswith('bastion0') From 77312085f8eee3fa14d760d4b46a9b950dd4320f Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 13 Aug 2014 21:20:25 +0000 Subject: [PATCH 011/193] lets add another dhcp net. not sure this will work --- .../dhcpd.conf.noc01.phx2.fedoraproject.org | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index e4ab136e4e..a8b9e19721 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -1732,3 +1732,24 @@ subnet 10.5.78.0 netmask 255.255.255.0 { } } + +subnet 10.5.131.0 netmask 255.255.255.0 { + allow booting; + allow bootp; + + option domain-name "mgmt.fedoraproject.org phx2.fedoraproject.org"; + option domain-name-servers 10.5.126.21, 10.5.126.22; + option routers 10.5.131.254; + option log-servers 10.5.126.29; + range 10.5.131.210 10.5.131.220; + next-server 10.5.126.41; + filename "pxelinux.0"; + + host retrace03 { + hardware ethernet ec:f4:bb:c0:38:a0; + fixed address 10.5.131.10; + option host-name "retrace03.qa.fedoraproject.org"; + next-server 10.5.126.41; + filename "pxelinux.0"; + } +} From ff28979c6b4c3fed0d9bd44ad965dbe157795937 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 13 Aug 2014 21:32:03 +0000 Subject: [PATCH 012/193] hey jude. did you know to put a - in it? --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index a8b9e19721..e9dd59c568 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -1747,7 +1747,7 @@ subnet 10.5.131.0 netmask 255.255.255.0 { host retrace03 { hardware ethernet ec:f4:bb:c0:38:a0; - fixed address 10.5.131.10; + fixed-address 10.5.131.10; option host-name "retrace03.qa.fedoraproject.org"; next-server 10.5.126.41; filename "pxelinux.0"; From 363292878a5cd8674791b44e6cc4635bde01aefd Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 13 Aug 2014 23:02:11 +0000 Subject: [PATCH 013/193] Set up fedmsg cert vars for the composer nodes. --- .../host_vars/branched-composer.phx2.fedoraproject.org | 9 +++++++++ .../host_vars/rawhide-composer.phx2.fedoraproject.org | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/inventory/host_vars/branched-composer.phx2.fedoraproject.org b/inventory/host_vars/branched-composer.phx2.fedoraproject.org index 51c4478456..9c31ec0196 100644 --- a/inventory/host_vars/branched-composer.phx2.fedoraproject.org +++ b/inventory/host_vars/branched-composer.phx2.fedoraproject.org @@ -3,3 +3,12 @@ vmhost: bvirthost08.phx2.fedoraproject.org eth0_ip: 10.5.125.66 eth1_ip: 10.5.127.53 volgroup: /dev/vg_bvirthost08 + +# These are consumed by a task in roles/fedmsg/base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: root +- service: bodhi + owner: root + group: masher diff --git a/inventory/host_vars/rawhide-composer.phx2.fedoraproject.org b/inventory/host_vars/rawhide-composer.phx2.fedoraproject.org index dbdc7ac344..0f26accad0 100644 --- a/inventory/host_vars/rawhide-composer.phx2.fedoraproject.org +++ b/inventory/host_vars/rawhide-composer.phx2.fedoraproject.org @@ -2,3 +2,12 @@ vmhost: bvirthost06.phx2.fedoraproject.org eth0_ip: 10.5.125.69 eth1_ip: 10.5.127.54 volgroup: /dev/vg_bvirthost06 + +# These are consumed by a task in roles/fedmsg/base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: root +- service: bodhi + owner: root + group: masher From 0d8757ee0e0d76ec06ba4116241e6ca3b53e8805 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 13 Aug 2014 23:45:41 +0000 Subject: [PATCH 014/193] and lo we may have a fix. --- .../dhcpd.conf.noc01.phx2.fedoraproject.org | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index e9dd59c568..7bebb4fe2c 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -164,8 +164,24 @@ subnet 10.5.124.128 netmask 255.255.255.128 { option routers 10.5.124.254; range 10.5.124.240 10.5.124.249; - next-server 10.5.126.41; - filename "pxelinux.0"; + + class "pxeclients" { + match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; + next-server 10.5.126.41; + + if option arch = 00:07 { + filename "EFI/shim.efi"; + } else { + filename "pxelinux.0"; + } + } + + host retrace03 { + hardware ethernet ec:f4:bb:c0:38:a0; + fixed-address 10.5.124.230; + option host-name "retrace03.qa.fedoraproject.org"; + } + host virthost-comm01 { hardware ethernet 5c:f3:fc:4d:db:fc; @@ -1745,11 +1761,4 @@ subnet 10.5.131.0 netmask 255.255.255.0 { next-server 10.5.126.41; filename "pxelinux.0"; - host retrace03 { - hardware ethernet ec:f4:bb:c0:38:a0; - fixed-address 10.5.131.10; - option host-name "retrace03.qa.fedoraproject.org"; - next-server 10.5.126.41; - filename "pxelinux.0"; - } } From 99638f4f80e944e2b7cd7fadc45c3ef6381d0d1b Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Thu, 14 Aug 2014 00:00:47 +0000 Subject: [PATCH 015/193] and dhcpd doesnt like the syntax from the manual. lovely --- .../files/dhcpd.conf.noc01.phx2.fedoraproject.org | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 7bebb4fe2c..658652c3aa 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -164,22 +164,14 @@ subnet 10.5.124.128 netmask 255.255.255.128 { option routers 10.5.124.254; range 10.5.124.240 10.5.124.249; - - class "pxeclients" { - match if substring (option vendor-class-identifier, 0, 9) = "PXEClient"; - next-server 10.5.126.41; - - if option arch = 00:07 { - filename "EFI/shim.efi"; - } else { - filename "pxelinux.0"; - } - } + next-server 10.5.126.41; host retrace03 { hardware ethernet ec:f4:bb:c0:38:a0; fixed-address 10.5.124.230; option host-name "retrace03.qa.fedoraproject.org"; + next-server 10.5.126.41; + filename "EFI/shim.efi"; } From 88fa1096992c7dd0a140954c6c4339675ed9266f Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 14 Aug 2014 07:27:45 +0000 Subject: [PATCH 016/193] buildmaster: fix variable typo in mail subject V2 --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index a28a5b80c8..1a50a61901 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s", + subject=Interpolate('%(prop:taskname)s')+" %(result)s in %(projectName)s on %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From 645e54eea975c995def45588f2be011625119650 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 14 Aug 2014 07:56:35 +0000 Subject: [PATCH 017/193] Another try --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 1a50a61901..8ff015d426 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject=Interpolate('%(prop:taskname)s')+" %(result)s in %(projectName)s on %(builder)s", + subject=Interpolate("%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s"), mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From 553269bf82918d779ea8f64b3cde6b95ec3d71f6 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 14 Aug 2014 08:13:48 +0000 Subject: [PATCH 018/193] buildmaster: revert taskname in the mail subject --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 8ff015d426..d12cb39d4a 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject=Interpolate("%(prop:taskname)s %(result)s in %(projectName)s on %(builder)s"), + subject="%(result)s in %(projectName)s on %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From da017ccc3907c83bc04f037f0d6701595cd8a2b9 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 14 Aug 2014 09:20:17 +0000 Subject: [PATCH 019/193] buildmaster: one last try putting taskname in the mail subject --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index d12cb39d4a..298bc60d60 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(result)s in %(projectName)s on %(builder)s", + subject="%(taskname)s %(result)s in %(projectName)s on %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From a8de08cac1c0c1e4b6978bebe85c7f4fd2b15a89 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 14 Aug 2014 11:36:32 +0000 Subject: [PATCH 020/193] buildmaster: no taskname in the mail subject for now --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 298bc60d60..d12cb39d4a 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(taskname)s %(result)s in %(projectName)s on %(builder)s", + subject="%(result)s in %(projectName)s on %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From 067fec4412b6f704d5495844e692dd86dcb9415d Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 14 Aug 2014 16:12:07 +0000 Subject: [PATCH 021/193] Copy in some handy badge admin scripts. --- .../badges/backend/files/grant-authorization | 76 +++++++++++++++ roles/badges/backend/files/revoke-badge | 94 +++++++++++++++++++ roles/badges/backend/tasks/main.yml | 2 + 3 files changed, 172 insertions(+) create mode 100644 roles/badges/backend/files/grant-authorization create mode 100644 roles/badges/backend/files/revoke-badge diff --git a/roles/badges/backend/files/grant-authorization b/roles/badges/backend/files/grant-authorization new file mode 100644 index 0000000000..71a44e148a --- /dev/null +++ b/roles/badges/backend/files/grant-authorization @@ -0,0 +1,76 @@ +#!/usr/bin/env python +""" +This is a CLI script for granting authorization on a single badge to somebody. +""" + +import __main__ +__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; +import pkg_resources +pkg_resources.require(__requires__) + +import argparse +import transaction +import sys + +from tahrir_api.dbapi import TahrirDatabase + +import fedmsg +import fedmsg.config + +import fedbadges.utils + + +def parse_args(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument('--user', default=None, help="A FAS username") + parser.add_argument('--badge', default=None, help="A badge id") + args = parser.parse_args() + if not args.user: + print "You must specify a FAS username." + sys.exit(1) + if not args.badge: + print "You must specify a badge id." + sys.exit(1) + return args + + +def initialize(): + fm_config = fedmsg.config.load_config() + fm_config['cert_prefix'] = 'fedbadges' + fm_config['name'] = 'relay_inbound' + fm_config['active'] = True + fedmsg.init(**fm_config) + uri = fm_config['badges_global']['database_uri'] + tahrir = TahrirDatabase( + uri, + notification_callback=fedbadges.utils.notification_callback, + ) + return tahrir + + +def main(tahrir, nickname, badge_id): + person = tahrir.get_person(nickname=nickname) + badge = tahrir.get_badge(badge_id) + + if not person: + print "No such person %r" % nickname + sys.exit(1) + + if not badge: + print "No such badge %r" % badge_id + sys.exit(1) + + print "granting", person.nickname, "rights to %r." % badge_id + try: + transaction.begin() + tahrir.add_authorization(badge_id, person.email) + transaction.commit() + except Exception as e: + transaction.abort() + print "Failure:", e + + +if __name__ == '__main__': + args = parse_args() + tahrir = initialize() + main(tahrir, args.user, args.badge) diff --git a/roles/badges/backend/files/revoke-badge b/roles/badges/backend/files/revoke-badge new file mode 100644 index 0000000000..816a27b093 --- /dev/null +++ b/roles/badges/backend/files/revoke-badge @@ -0,0 +1,94 @@ +#!/usr/bin/env python +""" This is a CLI script for revoking a single badge from a single person. + +The intent is to use it to batch revoke a badge from a list of people. +""" + +import __main__ +__main__.__requires__ = __requires__ = ["tahrir-api", "sqlalchemy>=0.7"]; +import pkg_resources +pkg_resources.require(__requires__) + +import argparse +import transaction +import sys + +from tahrir_api.dbapi import TahrirDatabase + +import fedmsg +import fedmsg.config + +import fedbadges.utils + + +def parse_args(): + parser = argparse.ArgumentParser(__doc__) + parser.add_argument('--user', default=None, help="A FAS username") + parser.add_argument('--badge', default=None, help="A badge id") + args = parser.parse_args() + if not args.user: + print "You must specify a FAS username." + sys.exit(1) + if not args.badge: + print "You must specify a badge id." + sys.exit(1) + return args + + +def initialize(): + fm_config = fedmsg.config.load_config() + fm_config['cert_prefix'] = 'fedbadges' + fm_config['name'] = 'relay_inbound' + fm_config['active'] = True + fedmsg.init(**fm_config) + uri = fm_config['badges_global']['database_uri'] + tahrir = TahrirDatabase( + uri, + notification_callback=fedbadges.utils.notification_callback, + ) + return tahrir + + +def main(tahrir, nickname, badge_id): + person = tahrir.get_person(nickname=nickname) + badge = tahrir.get_badge(badge_id) + + if not person: + print "No such person %r" % nickname + sys.exit(1) + + if not badge: + print "No such badge %r" % badge_id + sys.exit(1) + + already_has_it = [assertion.person for assertion in badge.assertions] + + if person not in already_has_it: + print "%r does not actually have the %r badge..." % (nickname, badge_id) + return + + print "removing", person.nickname, "from the %r badge." % badge_id + try: + transaction.begin() + to_delete = None + for assertion in person.assertions: + if assertion.badge == badge: + to_delete = assertion + break + + if to_delete: + #person.assertions.remove(to_delete) + tahrir.session.delete(to_delete) + tahrir.session.commit() + else: + raise ValueError("no such assertion found. weird.") + transaction.commit() + except Exception as e: + transaction.abort() + print "Failure:", e + + +if __name__ == '__main__': + args = parse_args() + tahrir = initialize() + main(tahrir, args.user, args.badge) diff --git a/roles/badges/backend/tasks/main.yml b/roles/badges/backend/tasks/main.yml index 09577ff55a..a01847c302 100644 --- a/roles/badges/backend/tasks/main.yml +++ b/roles/badges/backend/tasks/main.yml @@ -148,6 +148,8 @@ mode=750 with_items: - award-badge + - revoke-badge + - grant-authorization - get-badges-person-id tags: - scripts From 42ac4ae251d8092710a79580b6b701a00670899f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 14 Aug 2014 17:29:46 +0000 Subject: [PATCH 022/193] Change path to try something. --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 658652c3aa..7879e6958f 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -171,7 +171,7 @@ subnet 10.5.124.128 netmask 255.255.255.128 { fixed-address 10.5.124.230; option host-name "retrace03.qa.fedoraproject.org"; next-server 10.5.126.41; - filename "EFI/shim.efi"; + filename "uefi/shim.efi"; } From dd280ba8719e0553f53891a595457a02174c757f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 14 Aug 2014 18:39:12 +0000 Subject: [PATCH 023/193] Give more time to nrpe commands. --- roles/nagios_client/templates/nrpe.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nagios_client/templates/nrpe.cfg.j2 b/roles/nagios_client/templates/nrpe.cfg.j2 index be01de1de7..ca61669f1f 100644 --- a/roles/nagios_client/templates/nrpe.cfg.j2 +++ b/roles/nagios_client/templates/nrpe.cfg.j2 @@ -133,7 +133,7 @@ debug=0 # This specifies the maximum number of seconds that the NRPE daemon will # allow plugins to finish executing before killing them off. -command_timeout=60 +command_timeout=100 From 29eb5fa0efa22d071b720f7b44fd35d63666c1ac Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Thu, 14 Aug 2014 18:42:40 +0000 Subject: [PATCH 024/193] and we make sure there is a filename --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 7879e6958f..fae7fe3d76 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -165,6 +165,7 @@ subnet 10.5.124.128 netmask 255.255.255.128 { range 10.5.124.240 10.5.124.249; next-server 10.5.126.41; + filename "pxelinux.0"; host retrace03 { hardware ethernet ec:f4:bb:c0:38:a0; From 6a5cb53f2a05f9fa29e329a3aa20892a4b563507 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 15 Aug 2014 13:51:31 +0000 Subject: [PATCH 025/193] adamwill tells me I have the wrong hashtag. --- roles/badges/backend/files/cron/award-flock-paparazzi-badge | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/badges/backend/files/cron/award-flock-paparazzi-badge b/roles/badges/backend/files/cron/award-flock-paparazzi-badge index 928102d027..42b19b3d7d 100644 --- a/roles/badges/backend/files/cron/award-flock-paparazzi-badge +++ b/roles/badges/backend/files/cron/award-flock-paparazzi-badge @@ -197,11 +197,11 @@ def make_fas_cache(username, password): def get_persons(): for person in get_g_plus_persons('Fedora FLOCK'): yield person - for person in get_g_plus_persons('flock2fedora'): + for person in get_g_plus_persons('flocktofedora'): yield person for person in get_flickr_persons('fedora,flock'): yield person - for person in get_flickr_persons('flock2fedora'): + for person in get_flickr_persons('flocktofedora'): yield person From c64a788abe0a1a3b8fa36d9669f8fb222aed5aaa Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Fri, 15 Aug 2014 14:24:46 +0000 Subject: [PATCH 026/193] buildmaster: uncomment cat taskotron log step --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index d12cb39d4a..d6fbd02e7c 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -156,9 +156,8 @@ factory.addStep(ShellCommand(command=["runtask", '-i', Interpolate('%(prop:taskname)s.yml')], name='runtask')) -# disabling this for now until all the logging changes are pushed -# print the log (see T230) -#factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log")) +# capture the taskotron log +factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log")) from buildbot.config import BuilderConfig c['builders'] = [] From afc84323d2d09383b24f3c283ee3712591ba040b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Fri, 15 Aug 2014 18:20:04 +0000 Subject: [PATCH 027/193] value01 needs a hosts entry for koji to talk to it for koji plugin. --- roles/hosts/files/value01.phx2.fedoraproject.org-hosts | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 roles/hosts/files/value01.phx2.fedoraproject.org-hosts diff --git a/roles/hosts/files/value01.phx2.fedoraproject.org-hosts b/roles/hosts/files/value01.phx2.fedoraproject.org-hosts new file mode 100644 index 0000000000..1b3eaec0b3 --- /dev/null +++ b/roles/hosts/files/value01.phx2.fedoraproject.org-hosts @@ -0,0 +1,6 @@ +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +10.5.126.23 infrastructure.fedoraproject.org +10.5.126.23 puppet.fedoraproject.org puppet infrastructure.fedoraproject.org +10.5.126.52 admin.fedoraproject.org +10.5.125.59 koji.fedoraproject.org koji03 koji03.phx2.fedoraproject.org From 26789cb36dc4ef32b2af21d14c58c7156fe938df Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Fri, 15 Aug 2014 20:13:08 +0000 Subject: [PATCH 028/193] and maybe shared-networks will work? --- .../dhcpd.conf.noc01.phx2.fedoraproject.org | 693 +++++++++--------- 1 file changed, 352 insertions(+), 341 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index fae7fe3d76..563d313c3f 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -157,339 +157,6 @@ subnet 10.5.126.0 netmask 255.255.255.0 { filename "pxelinux.0"; } -subnet 10.5.124.128 netmask 255.255.255.128 { - option domain-name "qa.fedoraproject.org phx2.fedoraproject.org fedoraproject.org"; - option domain-name-servers 10.5.126.21, 10.5.126.22; - option log-servers 10.5.126.29; - option routers 10.5.124.254; - - range 10.5.124.240 10.5.124.249; - next-server 10.5.126.41; - filename "pxelinux.0"; - - host retrace03 { - hardware ethernet ec:f4:bb:c0:38:a0; - fixed-address 10.5.124.230; - option host-name "retrace03.qa.fedoraproject.org"; - next-server 10.5.126.41; - filename "uefi/shim.efi"; - } - - - host virthost-comm01 { - hardware ethernet 5c:f3:fc:4d:db:fc; - fixed-address 10.5.124.131; - option host-name "virthost-comm01"; - } - -# -# All staticly assigned, but listed here for completeness. -# -# 5C:F3:FC:85:64:34 - ppc-hub.qa.fedoraproject.org - 10.5.124.182 -# 5C:F3:FC:85:64:33 - ppc-builder1.qa.fedoraproject.org - 10.5.124.213 -# 5C:F3:FC:85:64:36 - ppc-builder2.qa.fedoraproject.org - 10.5.124.214 -# 5C:F3:FC:85:64:37 - ppc-builder3.qa.fedoraproject.org - 10.5.124.215 -# 5C:F3:FC:85:64:38 - ppc-builder4.qa.fedoraproject.org - 10.5.124.216 -# 5C:F3:FC:85:64:35 - ppc-composer.qa.fedoraproject.org - 10.5.124.217 - - host ppc-comm01-mgmt { - hardware ethernet 5c:f3:fc:2e:93:72; - fixed-address 10.5.124.231; - option host-name "ppc-comm01-mgmt"; - } - - host ppc-comm01 { - hardware ethernet 6c:ae:8b:00:0f:f0; - fixed-address 10.5.124.219; - option host-name "ppc-comm01"; - filename "yaboot"; - next-server 10.5.126.41; - } - - - host qa01 { - hardware ethernet 00:21:5E:C7:5C:84; - fixed-address 10.5.124.151; - } - - host qa02 { - hardware ethernet 00:21:5e:c6:cc:9c; - fixed-address 10.5.124.152; - } - - host qa03 { - hardware ethernet 00:21:5E:C6:CD:48; - fixed-address 10.5.124.153; - } - - host qa04 { - hardware ethernet 00:21:5E:C7:2A:1C; - fixed-address 10.5.124.154; - } - - host qa05 { - hardware ethernet 00:21:5E:C7:5F:04; - fixed-address 10.5.124.155; - } - - host qa06 { - hardware ethernet 00:21:5E:C6:57:08; - fixed-address 10.5.124.156; - } - - host qa07 { - hardware ethernet E4:1F:13:E5:53:94; - fixed-address 10.5.124.157; - } - - host qa08 { - hardware ethernet E4:1F:13:E5:46:80; - fixed-address 10.5.124.158; - } - - host virt01 { - hardware ethernet 52:54:00:a2:de:30; - fixed-address 10.5.124.159; - option host-name "virt01"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt02 { - hardware ethernet 52:54:00:fe:22:ff; - fixed-address 10.5.124.160; - option host-name "virt02"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt03 { - hardware ethernet 52:54:00:c5:04:14; - fixed-address 10.5.124.161; - option host-name "virt03"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt04 { - hardware ethernet 52:54:00:b5:97:30; - fixed-address 10.5.124.162; - option host-name "virt04"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt05 { - hardware ethernet 52:54:00:9a:25:d3; - fixed-address 10.5.124.163; - option host-name "virt05"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt06 { - hardware ethernet 52:54:00:78:ae:44; - fixed-address 10.5.124.164; - option host-name "virt06"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt07 { - hardware ethernet 52:54:00:00:eb:e9; - fixed-address 10.5.124.165; - option host-name "virt07"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt08 { - hardware ethernet 52:54:00:24:dd:72; - fixed-address 10.5.124.166; - option host-name "virt08"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt09 { - hardware ethernet 52:54:00:a1:a0:23; - fixed-address 10.5.124.167; - option host-name "virt09"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt10 { - hardware ethernet 52:54:00:88:a2:9d; - fixed-address 10.5.124.168; - option host-name "virt10"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt11 { - hardware ethernet 52:54:00:1c:de:bf; - fixed-address 10.5.124.169; - option host-name "virt11"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt12 { - hardware ethernet 52:54:00:28:90:26; - fixed-address 10.5.124.170; - option host-name "virt12"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt13 { - hardware ethernet 52:54:00:2c:5e:61; - fixed-address 10.5.124.192; - option host-name "virt13"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt14 { - hardware ethernet 52:54:00:45:7f:9d; - fixed-address 10.5.124.193; - option host-name "virt14"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt15 { - hardware ethernet 52:54:00:1d:15:85; - fixed-address 10.5.124.194; - option host-name "virt15"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - next-server 10.5.124.228; - filename "pxelinux.0"; - } - - host virt16 { - hardware ethernet 52:54:00:f2:cc:2a; - fixed-address 10.5.124.195; - option host-name "virt16"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - next-server 10.5.124.228; - filename "pxelinux.0"; - } - - host virt17 { - hardware ethernet 52:54:00:58:9b:0e; - fixed-address 10.5.124.196; - option host-name "virt17"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - next-server 10.5.124.228; - filename "pxelinux.0"; - } - - host virt18 { - hardware ethernet 52:54:00:22:3b:07; - fixed-address 10.5.124.197; - option host-name "virt18"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - next-server 10.5.124.228; - filename "pxelinux.0"; - } - - host virt19 { - hardware ethernet 52:54:00:27:35:92; - fixed-address 10.5.124.198; - option host-name "virt19"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt20 { - hardware ethernet 52:54:00:60:97:00; - fixed-address 10.5.124.199; - option host-name "virt20"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt21 { - hardware ethernet 52:54:00:58:76:db; - fixed-address 10.5.124.200; - option host-name "virt21"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt22 { - hardware ethernet 52:54:00:41:5a:1a; - fixed-address 10.5.124.183; - option host-name "virt22"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt23 { - hardware ethernet 52:54:00:7c:79:63; - fixed-address 10.5.124.184; - option host-name "virt23"; - option ntp-servers 66.187.233.4,192.43.244.18,128.118.25.5,204.152.184.72; - } - - host virt24 { - hardware ethernet 52:54:00:8d:7d:96; - fixed-address 10.5.124.185; - option host-name "virt24"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt25 { - hardware ethernet 52:54:00:f4:76:92; - fixed-address 10.5.124.186; - option host-name "virt25"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt26 { - hardware ethernet 52:54:00:76:a4:31; - fixed-address 10.5.124.187; - option host-name "virt26"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt27 { - hardware ethernet 52:54:00:bc:e8:47; - fixed-address 10.5.124.188; - option host-name "virt27"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt28 { - hardware ethernet 52:54:00:3f:eb:29; - fixed-address 10.5.124.189; - option host-name "virt28"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host virt29 { - hardware ethernet 52:54:00:ab:7f:ac; - fixed-address 10.5.124.190; - option host-name "virt29"; - option ntp-servers 66.18.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } - - host retrace01 { - hardware ethernet e4:1f:13:ba:ad:88; - fixed-address 10.5.124.171; - next-server 10.5.126.41; - option host-name "retrace01"; - filename "pxelinux.0"; - } - - host kernel01 { - hardware ethernet 90:b1:1c:31:c9:ca; - fixed-address 10.5.124.173; - option host-name "kernel01"; - filename "pxelinux.0"; - } - - host kernel02 { - hardware ethernet 90:b1:1c:31:bf:27; - fixed-address 10.5.124.175; - option host-name "kernel02"; - filename "pxelinux.0"; - } - - - host cosmos01 { - hardware ethernet 40:f2:e9:1b:a7:98; - fixed-address 10.5.124.134; - option host-name "cosmos01"; - filename "pxelinux.0"; - } - -} - subnet 10.5.127.0 netmask 255.255.255.0 { allow booting; allow bootp; @@ -1742,16 +1409,360 @@ subnet 10.5.78.0 netmask 255.255.255.0 { } -subnet 10.5.131.0 netmask 255.255.255.0 { - allow booting; - allow bootp; - option domain-name "mgmt.fedoraproject.org phx2.fedoraproject.org"; +shared-network qa { + option domain-name "qa.fedoraproject.org phx2.fedoraproject.org fedoraproject.org"; option domain-name-servers 10.5.126.21, 10.5.126.22; - option routers 10.5.131.254; option log-servers 10.5.126.29; - range 10.5.131.210 10.5.131.220; - next-server 10.5.126.41; - filename "pxelinux.0"; + + subnet 10.5.124.128 netmask 255.255.255.128 { + + option routers 10.5.124.254; + + range 10.5.124.240 10.5.124.249; + next-server 10.5.126.41; + filename "pxelinux.0"; + + host retrace03 { + hardware ethernet ec:f4:bb:c0:38:a0; + fixed-address 10.5.124.230; + option host-name "retrace03.qa.fedoraproject.org"; + next-server 10.5.126.41; + filename "uefi/shim.efi"; + } + + + host virthost-comm01 { + hardware ethernet 5c:f3:fc:4d:db:fc; + fixed-address 10.5.124.131; + option host-name "virthost-comm01"; + } + +# +# All staticly assigned, but listed here for completeness. +# +# 5C:F3:FC:85:64:34 - ppc-hub.qa.fedoraproject.org - 10.5.124.182 +# 5C:F3:FC:85:64:33 - ppc-builder1.qa.fedoraproject.org - 10.5.124.213 +# 5C:F3:FC:85:64:36 - ppc-builder2.qa.fedoraproject.org - 10.5.124.214 +# 5C:F3:FC:85:64:37 - ppc-builder3.qa.fedoraproject.org - 10.5.124.215 +# 5C:F3:FC:85:64:38 - ppc-builder4.qa.fedoraproject.org - 10.5.124.216 +# 5C:F3:FC:85:64:35 - ppc-composer.qa.fedoraproject.org - 10.5.124.217 + + host ppc-comm01-mgmt { + hardware ethernet 5c:f3:fc:2e:93:72; + fixed-address 10.5.124.231; + option host-name "ppc-comm01-mgmt"; + } + + host ppc-comm01 { + hardware ethernet 6c:ae:8b:00:0f:f0; + fixed-address 10.5.124.219; + option host-name "ppc-comm01"; + filename "yaboot"; + next-server 10.5.126.41; + } + + + host qa01 { + hardware ethernet 00:21:5E:C7:5C:84; + fixed-address 10.5.124.151; + } + + host qa02 { + hardware ethernet 00:21:5e:c6:cc:9c; + fixed-address 10.5.124.152; + } + + host qa03 { + hardware ethernet 00:21:5E:C6:CD:48; + fixed-address 10.5.124.153; + } + + host qa04 { + hardware ethernet 00:21:5E:C7:2A:1C; + fixed-address 10.5.124.154; + } + + host qa05 { + hardware ethernet 00:21:5E:C7:5F:04; + fixed-address 10.5.124.155; + } + + host qa06 { + hardware ethernet 00:21:5E:C6:57:08; + fixed-address 10.5.124.156; + } + + host qa07 { + hardware ethernet E4:1F:13:E5:53:94; + fixed-address 10.5.124.157; + } + + host qa08 { + hardware ethernet E4:1F:13:E5:46:80; + fixed-address 10.5.124.158; + } + + host virt01 { + hardware ethernet 52:54:00:a2:de:30; + fixed-address 10.5.124.159; + option host-name "virt01"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt02 { + hardware ethernet 52:54:00:fe:22:ff; + fixed-address 10.5.124.160; + option host-name "virt02"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt03 { + hardware ethernet 52:54:00:c5:04:14; + fixed-address 10.5.124.161; + option host-name "virt03"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt04 { + hardware ethernet 52:54:00:b5:97:30; + fixed-address 10.5.124.162; + option host-name "virt04"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt05 { + hardware ethernet 52:54:00:9a:25:d3; + fixed-address 10.5.124.163; + option host-name "virt05"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt06 { + hardware ethernet 52:54:00:78:ae:44; + fixed-address 10.5.124.164; + option host-name "virt06"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt07 { + hardware ethernet 52:54:00:00:eb:e9; + fixed-address 10.5.124.165; + option host-name "virt07"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt08 { + hardware ethernet 52:54:00:24:dd:72; + fixed-address 10.5.124.166; + option host-name "virt08"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt09 { + hardware ethernet 52:54:00:a1:a0:23; + fixed-address 10.5.124.167; + option host-name "virt09"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt10 { + hardware ethernet 52:54:00:88:a2:9d; + fixed-address 10.5.124.168; + option host-name "virt10"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt11 { + hardware ethernet 52:54:00:1c:de:bf; + fixed-address 10.5.124.169; + option host-name "virt11"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt12 { + hardware ethernet 52:54:00:28:90:26; + fixed-address 10.5.124.170; + option host-name "virt12"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt13 { + hardware ethernet 52:54:00:2c:5e:61; + fixed-address 10.5.124.192; + option host-name "virt13"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt14 { + hardware ethernet 52:54:00:45:7f:9d; + fixed-address 10.5.124.193; + option host-name "virt14"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt15 { + hardware ethernet 52:54:00:1d:15:85; + fixed-address 10.5.124.194; + option host-name "virt15"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + next-server 10.5.124.228; + filename "pxelinux.0"; + } + + host virt16 { + hardware ethernet 52:54:00:f2:cc:2a; + fixed-address 10.5.124.195; + option host-name "virt16"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + next-server 10.5.124.228; + filename "pxelinux.0"; + } + + host virt17 { + hardware ethernet 52:54:00:58:9b:0e; + fixed-address 10.5.124.196; + option host-name "virt17"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + next-server 10.5.124.228; + filename "pxelinux.0"; + } + + host virt18 { + hardware ethernet 52:54:00:22:3b:07; + fixed-address 10.5.124.197; + option host-name "virt18"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + next-server 10.5.124.228; + filename "pxelinux.0"; + } + + host virt19 { + hardware ethernet 52:54:00:27:35:92; + fixed-address 10.5.124.198; + option host-name "virt19"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt20 { + hardware ethernet 52:54:00:60:97:00; + fixed-address 10.5.124.199; + option host-name "virt20"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt21 { + hardware ethernet 52:54:00:58:76:db; + fixed-address 10.5.124.200; + option host-name "virt21"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt22 { + hardware ethernet 52:54:00:41:5a:1a; + fixed-address 10.5.124.183; + option host-name "virt22"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt23 { + hardware ethernet 52:54:00:7c:79:63; + fixed-address 10.5.124.184; + option host-name "virt23"; + option ntp-servers 66.187.233.4,192.43.244.18,128.118.25.5,204.152.184.72; + } + + host virt24 { + hardware ethernet 52:54:00:8d:7d:96; + fixed-address 10.5.124.185; + option host-name "virt24"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt25 { + hardware ethernet 52:54:00:f4:76:92; + fixed-address 10.5.124.186; + option host-name "virt25"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt26 { + hardware ethernet 52:54:00:76:a4:31; + fixed-address 10.5.124.187; + option host-name "virt26"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt27 { + hardware ethernet 52:54:00:bc:e8:47; + fixed-address 10.5.124.188; + option host-name "virt27"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt28 { + hardware ethernet 52:54:00:3f:eb:29; + fixed-address 10.5.124.189; + option host-name "virt28"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host virt29 { + hardware ethernet 52:54:00:ab:7f:ac; + fixed-address 10.5.124.190; + option host-name "virt29"; + option ntp-servers 66.18.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } + + host retrace01 { + hardware ethernet e4:1f:13:ba:ad:88; + fixed-address 10.5.124.171; + next-server 10.5.126.41; + option host-name "retrace01"; + filename "pxelinux.0"; + } + + host kernel01 { + hardware ethernet 90:b1:1c:31:c9:ca; + fixed-address 10.5.124.173; + option host-name "kernel01"; + filename "pxelinux.0"; + } + + host kernel02 { + hardware ethernet 90:b1:1c:31:bf:27; + fixed-address 10.5.124.175; + option host-name "kernel02"; + filename "pxelinux.0"; + } + + + host cosmos01 { + hardware ethernet 40:f2:e9:1b:a7:98; + fixed-address 10.5.124.134; + option host-name "cosmos01"; + filename "pxelinux.0"; + } + + } + + + subnet 10.5.131.0 netmask 255.255.255.0 { + option routers 10.5.131.254; + range 10.5.131.240 10.5.131.249; + next-server 10.5.126.41; + filename "pxelinux.0"; + + host retrace04 { + hardware ethernet ec:f4:bb:c0:46:38; + fixed-address 10.5.131.14; + option host-name "retrace04.qa.fedoraproject.org"; + next-server 10.5.126.41; + filename "pxelinux.0"; + } + } } + + + From ffcd91be656e5f5411fd29f3ee912a8e8d4cd2a5 Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Fri, 15 Aug 2014 20:19:01 +0000 Subject: [PATCH 029/193] use sbt-extras instead of sbt Signed-off-by: Ricky Elrod --- files/jenkins/slaves/sbt.repo | 10 +++++----- playbooks/groups/jenkins-cloud.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/files/jenkins/slaves/sbt.repo b/files/jenkins/slaves/sbt.repo index 02af2a2de9..40cd3e8223 100644 --- a/files/jenkins/slaves/sbt.repo +++ b/files/jenkins/slaves/sbt.repo @@ -1,6 +1,6 @@ -[sbt-fedorapeople] -name=SBT Fedorapeople Repo -baseurl=http://repos.fedorapeople.org/repos/codeblock/sbt/fedora-18/RPMS/ -enabled=1 -skip_if_unavailable=1 +[codeblock-sbt-extras] +name=Copr repo for sbt-extras owned by codeblock +baseurl=http://copr-be.cloud.fedoraproject.org/results/codeblock/sbt-extras/fedora-$releasever-$basearch/ +skip_if_unavailable=True gpgcheck=0 +enabled=1 diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 81183a2c1c..ed318f2bff 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -346,7 +346,7 @@ - python3 - python-nose-cover3 - python3-nose-cover3 - - sbt + - sbt-extras - glibc.i686 - glibc-devel.i686 - libstdc++.i686 From 4941a44e59d9de89f437e6ec825f5510f9be4ccd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 15 Aug 2014 23:55:34 +0200 Subject: [PATCH 030/193] Run fas_client before nagios_client --- playbooks/groups/fedimg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/fedimg.yml b/playbooks/groups/fedimg.yml index 78162f356e..03e7079627 100644 --- a/playbooks/groups/fedimg.yml +++ b/playbooks/groups/fedimg.yml @@ -30,9 +30,9 @@ - base - rkhunter - { role: denyhosts, when: ansible_distribution_major_version != '7' } + - fas_client - nagios_client - hosts - - fas_client - collectd/base - fedmsg/base - sudo From 98ec937ba2966f3d2b9d3417f1437e5d0f97a987 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Sat, 16 Aug 2014 16:16:17 +0000 Subject: [PATCH 031/193] update to use a more direct route for sms for me --- roles/nagios_server/files/nagios-external/contacts/ausil.cfg | 4 ++-- roles/nagios_server/files/nagios/contacts/ausil.cfg | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/nagios_server/files/nagios-external/contacts/ausil.cfg b/roles/nagios_server/files/nagios-external/contacts/ausil.cfg index a8afa7476c..c025f731a4 100644 --- a/roles/nagios_server/files/nagios-external/contacts/ausil.cfg +++ b/roles/nagios_server/files/nagios-external/contacts/ausil.cfg @@ -10,7 +10,7 @@ define contact{ #service_notification_commands notify-by-epager #host_notification_commands host-notify-by-epager #email ausil@fedoraproject.org - pager mobile@ausil.us - email mobile@ausil.us + pager 3098682442@tmomail.net + email 3098682442@tmomail.net } diff --git a/roles/nagios_server/files/nagios/contacts/ausil.cfg b/roles/nagios_server/files/nagios/contacts/ausil.cfg index a8afa7476c..c025f731a4 100644 --- a/roles/nagios_server/files/nagios/contacts/ausil.cfg +++ b/roles/nagios_server/files/nagios/contacts/ausil.cfg @@ -10,7 +10,7 @@ define contact{ #service_notification_commands notify-by-epager #host_notification_commands host-notify-by-epager #email ausil@fedoraproject.org - pager mobile@ausil.us - email mobile@ausil.us + pager 3098682442@tmomail.net + email 3098682442@tmomail.net } From 249f1051dc1ff5a2af0350e9c4556ab6e9b4ab21 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Sat, 16 Aug 2014 17:12:45 +0000 Subject: [PATCH 032/193] Add sysadmin-noc here too. --- inventory/host_vars/log01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/host_vars/log01.phx2.fedoraproject.org b/inventory/host_vars/log01.phx2.fedoraproject.org index 252996978b..f76de796d8 100644 --- a/inventory/host_vars/log01.phx2.fedoraproject.org +++ b/inventory/host_vars/log01.phx2.fedoraproject.org @@ -20,4 +20,4 @@ mem_size: 16384 num_cpus: 16 host_backup_targets: ['/var/log'] -fas_client_groups: fi-apprentice,sysadmin-logs +fas_client_groups: fi-apprentice,sysadmin-logs,sysadmin-noc From baa5b40a770a5ad7cc6d9f844253ee2bfdc6652b Mon Sep 17 00:00:00 2001 From: Ricky Elrod Date: Sun, 17 Aug 2014 20:06:12 +0000 Subject: [PATCH 033/193] try fixing selinux for nagios hostname access Signed-off-by: Ricky Elrod --- roles/nagios_server/files/selinux/hostname.pp | Bin 0 -> 1010 bytes roles/nagios_server/files/selinux/hostname.te | 11 +++++++++++ roles/nagios_server/tasks/main.yml | 18 +++++++++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 roles/nagios_server/files/selinux/hostname.pp create mode 100644 roles/nagios_server/files/selinux/hostname.te diff --git a/roles/nagios_server/files/selinux/hostname.pp b/roles/nagios_server/files/selinux/hostname.pp new file mode 100644 index 0000000000000000000000000000000000000000..0db5752ea007f255235e8e3c9522e7360340de6b GIT binary patch literal 1010 zcmb_aK~4iP3^Wu(Z-p1o7a;fn7jE>_2SlMYWl>40U9Z$bzkqmePeoHQlC^3i4jm~< z)*joF?ED;d`wJ1lx)qV{!)`wjk@?H?wMqSUI?Ky`t(PKlS7>AT1NM*HS&Cme5|O8e z$LEsrVE3yM#P@Ysz>6nB{ZVCFhk10+L$_UAR=!x zaUulb4CJ>@U07(n!rGbedC0+HPtE{5FU3`pyI^=j@M}CiT);(wabyi_!|MQy0g!;j zJa;ec6b}*^_h9dPA1v@a|1)>U!EXYFeTZ5dnufiouU2!e__=^Pz^X7k- Date: Mon, 18 Aug 2014 07:33:34 +0000 Subject: [PATCH 034/193] changing buildmaster role to work with local database when needed --- inventory/group_vars/qadevel-stg | 2 +- roles/taskotron/buildmaster/tasks/main.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index 256a69f943..c5017cc3fc 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -19,7 +19,7 @@ virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ m hostname={{ inventory_hostname }}" --network=bridge=br0 --autostart --noautoconsole -buildmaster_db_name: buildmaster_ci_stg +buildmaster_db_host: localhost buildmaster_template: ci.master.cfg.j2 buildmaster_endpoint: taskmaster buildslave_ssh_pubkey: '' diff --git a/roles/taskotron/buildmaster/tasks/main.yml b/roles/taskotron/buildmaster/tasks/main.yml index 11c9a8a77e..6b798d8761 100644 --- a/roles/taskotron/buildmaster/tasks/main.yml +++ b/roles/taskotron/buildmaster/tasks/main.yml @@ -17,7 +17,8 @@ notify: - restart httpd -- name: ensure dev database is created +- name: ensure buildmaster database is created + when: buildmaster_db_host != 'localhost' delegate_to: "{{ buildmaster_db_host }}" sudo: true sudo_user: postgres From 633423c5ad2e0571a730498fec6a11f700bc897f Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 07:37:18 +0000 Subject: [PATCH 035/193] change deployment type for qadevel-stg to not conflict with taskotron stg --- inventory/group_vars/qadevel-stg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index c5017cc3fc..5f63d78181 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -30,5 +30,5 @@ buildslave_poll_interval: 1800 master_dir: /home/buildmaster/master master_user: buildmaster external_hostname: qadevel-stg.qa.fedoraproject.org -deployment_type: stg +deployment_type: qadevel-stg tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] From 962cbdc3dd81ccddd581ee48237f9c4966d9a4bd Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 07:43:23 +0000 Subject: [PATCH 036/193] adding missing import to qadevel buildmaster config --- roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 index d1bf48e86e..379a6be09f 100644 --- a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -118,6 +118,7 @@ c['schedulers'].append(ForceScheduler( from buildbot.process.factory import BuildFactory from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand +from buildbot.process.properties import Interpolate def create_test_factory(repo_name, func=True): From e2f60001d3fbff7ffd33963a4da3153bcdcf2129 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 08:10:19 +0000 Subject: [PATCH 037/193] creating rest of qadevel playbook --- inventory/group_vars/qadevel | 14 ++++ playbooks/groups/qadevel.yml | 70 +++++++++++++++++++ .../templates/ci.master.cfg.j2 | 4 +- 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 playbooks/groups/qadevel.yml diff --git a/inventory/group_vars/qadevel b/inventory/group_vars/qadevel index 64b9d921c4..390f91d8fd 100644 --- a/inventory/group_vars/qadevel +++ b/inventory/group_vars/qadevel @@ -18,3 +18,17 @@ virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ m gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 hostname={{ inventory_hostname }}" --network=bridge=br0 --autostart --noautoconsole + +buildmaster_db_host: localhost +buildmaster_template: ci.master.cfg.j2 +buildmaster_endpoint: buildmaster +buildslave_ssh_pubkey: '' +buildslave_port: 9989 +buildmaster_dir: /home/buildmaster/master +buildslave_dir: /home/buildslave/slave +buildslave_poll_interval: 1800 +master_dir: /home/buildmaster/master +master_user: buildmaster +external_hostname: qadevel.qa.fedoraproject.org +deployment_type: qadevel +tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] diff --git a/playbooks/groups/qadevel.yml b/playbooks/groups/qadevel.yml new file mode 100644 index 0000000000..3c9f937597 --- /dev/null +++ b/playbooks/groups/qadevel.yml @@ -0,0 +1,70 @@ +--- +# create a new qadevel server +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/mirrorlist or from hostvars + +- name: make taskotron-ci staging + hosts: qadevel + user: root + gather_facts: False + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: qadevel + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: base, tags:['base'] } + - { role: rkhunter, tags:['rkhunter'] } + - { role: nagios_client, tags:['nagios_client'] } + - hosts + - { role: fas_client, tags:['fas_client'] } + - { role: collectd/base, tags:['collectd_base'] } + - { role: yum-cron, tags:['yumcron'] } + - { role: sudo, tags:['sudo'] } + + tasks: + # this is how you include other task lists + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/apache.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: configure taskotron-ci master + hosts: qadevel + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: taskotron/buildmaster, tags: ['buildmaster'] } + - { role: taskotron/buildmaster-configure, tags: ['buildmasterconfig'] } + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 index 379a6be09f..e2ec5ac8a6 100644 --- a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -16,10 +16,10 @@ c = BuildmasterConfig = {} from buildbot.buildslave import BuildSlave c['slaves'] = [ {% for buildslave in buildslaves %} - {% if deployment_type == 'stg' %} + {% if deployment_type == 'qadevel-stg' %} BuildSlave("{{ buildslave }}", "{{ qadevel_stg_buildslave_password }}"), {% endif %} - {% if deployment_type == 'prod' %} + {% if deployment_type == 'qadevel-prod' %} BuildSlave("{{ buildslave }}", "{{ qadevel_prod_buildslave_password }}"), {% endif %} {% endfor %} From a0df241c8540b55ee08dd1259e8de55c9628bff4 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:16:20 +0000 Subject: [PATCH 038/193] fixing missing data from qadevel host vars --- .../host_vars/qadevel.qa.fedoraproject.org | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/inventory/host_vars/qadevel.qa.fedoraproject.org b/inventory/host_vars/qadevel.qa.fedoraproject.org index 6efb4b5d79..0b97ec82fc 100644 --- a/inventory/host_vars/qadevel.qa.fedoraproject.org +++ b/inventory/host_vars/qadevel.qa.fedoraproject.org @@ -8,3 +8,21 @@ volgroup: /dev/Guests00 eth0_ip: 10.5.124.180 vmhost: virthost-comm01.qa.fedoraproject.org datacenter: phx2 + +fas_client_groups: sysadmin-qa,sysadmin-main + +# default virt install command is for a single nic-device +# define in another group file for more nics (see buildvm) +virt_install_command: /usr/sbin/virt-install -n {{ inventory_hostname }} -r {{ mem_size }} + --disk bus=virtio,path={{ volgroup }}/{{ inventory_hostname }} + --vcpus={{ num_cpus }} -l {{ ks_repo }} -x + "ks={{ ks_url }} ip={{ eth0_ip }} netmask={{ nm }} + gateway={{ gw }} dns={{ dns }} console=tty0 console=ttyS0 + hostname={{ inventory_hostname }}" + --network=bridge=br0 --autostart --noautoconsole + +public_hostname: qadevel.qa.fedoraproject.org +buildmaster: 10.5.124.180 + +buildslaves: + - qadevel From 0a5d80977f861e856e9e30e6b8cfb6e8242f68f3 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:18:14 +0000 Subject: [PATCH 039/193] changing deployment type to match buildmaster configuration --- inventory/group_vars/qadevel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/qadevel b/inventory/group_vars/qadevel index 390f91d8fd..2070fc135e 100644 --- a/inventory/group_vars/qadevel +++ b/inventory/group_vars/qadevel @@ -30,5 +30,5 @@ buildslave_poll_interval: 1800 master_dir: /home/buildmaster/master master_user: buildmaster external_hostname: qadevel.qa.fedoraproject.org -deployment_type: qadevel +deployment_type: qadevel-prod tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] From a766e53cadfcc266b4a4dad28e09a2b51b33909b Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 18 Aug 2014 09:16:49 +0000 Subject: [PATCH 040/193] buildmaster: put more information into the email reports --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index d6fbd02e7c..ee45fcac38 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -269,15 +269,16 @@ def html_message_formatter(mode, name, build, results, master_status): text.append(u'') text.append(u'

') text.append(u'-The Buildbot') + taskname = re.search("task=\['(.*)\.yml'\]", t).group(1) + test.insert(0, '

Task name: %s

' % taskname) return { 'body': u"\n".join(text), 'type': 'html' } - mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(result)s in %(projectName)s on %(builder)s", + subject="%(projectName)s %(result)s on " + deployment_type + " %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From ab85e701df86c0c60ccf20825a69d0b217562ec3 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 18 Aug 2014 09:26:47 +0000 Subject: [PATCH 041/193] buildmaster: fix variable --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index ee45fcac38..edab2832d2 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,7 +278,7 @@ def html_message_formatter(mode, name, build, results, master_status): mn = MailNotifier(fromaddr='taskotron@fedoraproject.org', sendToInterestedUsers=False, - subject="%(projectName)s %(result)s on " + deployment_type + " %(builder)s", + subject="%(projectName)s %(result)s on {{ deployment_type }} %(builder)s", mode=('failing', 'exception', 'warnings'), extraRecipients=['qa-taskotron-admin-members@fedoraproject.org'], relayhost="bastion.phx2.fedoraproject.org", From 4d439b477091d8a2ba36fd69905f4e64126f25ea Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:27:24 +0000 Subject: [PATCH 042/193] adding local buildslave for testing qadevel buildbot --- playbooks/groups/qadevel.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/playbooks/groups/qadevel.yml b/playbooks/groups/qadevel.yml index 3c9f937597..74025dafac 100644 --- a/playbooks/groups/qadevel.yml +++ b/playbooks/groups/qadevel.yml @@ -68,3 +68,21 @@ handlers: - include: "{{ handlers }}/restart_services.yml" + +- name: configure taskotron-ci local slave + hosts: qadevel + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - { role: taskotron/buildslave, tags: ['buildslave'] } + - { role: taskotron/buildslave-configure, tags: ['buildslaveconfig'] } + + handlers: + - include: "{{ handlers }}/restart_services.yml" From c0f38ccb00f7c5d97023961ee95559af41ad29cf Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:38:22 +0000 Subject: [PATCH 043/193] adding buildslave vars to qadevel --- inventory/group_vars/qadevel | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inventory/group_vars/qadevel b/inventory/group_vars/qadevel index 2070fc135e..de0aea8bb9 100644 --- a/inventory/group_vars/qadevel +++ b/inventory/group_vars/qadevel @@ -32,3 +32,8 @@ master_user: buildmaster external_hostname: qadevel.qa.fedoraproject.org deployment_type: qadevel-prod tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] + +# for now, we're just doing a local slave so we need the slave vars in here +slave_home: /home/buildslave/ +slave_dir: /home/buildslave/slave +slave_user: buildslave From 839e4a0c4d778d88736e4661ed52f42afd9c6f76 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:44:16 +0000 Subject: [PATCH 044/193] adding required buildslave host var to qadevel --- inventory/host_vars/qadevel.qa.fedoraproject.org | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inventory/host_vars/qadevel.qa.fedoraproject.org b/inventory/host_vars/qadevel.qa.fedoraproject.org index 0b97ec82fc..718de4d03c 100644 --- a/inventory/host_vars/qadevel.qa.fedoraproject.org +++ b/inventory/host_vars/qadevel.qa.fedoraproject.org @@ -26,3 +26,6 @@ buildmaster: 10.5.124.180 buildslaves: - qadevel + +# needed for local buildslave +buildslave_name: 'qadevel' From 2cd2014c64b677b5be6c3ca6a6a5d94ae136199a Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:46:53 +0000 Subject: [PATCH 045/193] we don't need buildslave ssh keys for qadevel --- roles/taskotron/buildslave-configure/tasks/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/taskotron/buildslave-configure/tasks/main.yml b/roles/taskotron/buildslave-configure/tasks/main.yml index 830140749f..1be1d37f0a 100644 --- a/roles/taskotron/buildslave-configure/tasks/main.yml +++ b/roles/taskotron/buildslave-configure/tasks/main.yml @@ -17,15 +17,19 @@ - host - name: create slave ssh directory + when: deployment_type != qadevel file: path={{ slave_home }}/.ssh owner={{ slave_user }} group={{ slave_user }} mode=0700 state=directory - name: install slave ssh private key + when: deployment_type != qadevel copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa owner={{ slave_user }} group={{ slave_user }} mode=0600 - name: install slave ssh public key + when: deployment_type != qadevel copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa.pub owner={{ slave_user }} group={{ slave_user }} mode=0644 - name: make sure master is in known_hosts + when: deployment_type != qadevel lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }} - name: generate buildslave service file From 4d83c27e6abce3ff1a029a37a50704309f11d0d3 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:50:06 +0000 Subject: [PATCH 046/193] use valid ansible syntax to determine when to use buildslave ssh keypair --- roles/taskotron/buildslave-configure/tasks/main.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/taskotron/buildslave-configure/tasks/main.yml b/roles/taskotron/buildslave-configure/tasks/main.yml index 1be1d37f0a..9ee7692d30 100644 --- a/roles/taskotron/buildslave-configure/tasks/main.yml +++ b/roles/taskotron/buildslave-configure/tasks/main.yml @@ -17,19 +17,19 @@ - host - name: create slave ssh directory - when: deployment_type != qadevel + when: buildslave_public_sshkey_file is defined file: path={{ slave_home }}/.ssh owner={{ slave_user }} group={{ slave_user }} mode=0700 state=directory - name: install slave ssh private key - when: deployment_type != qadevel + when: buildslave_private_sshkey_file is defined copy: src={{ private }}/files/taskotron/{{ buildslave_private_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa owner={{ slave_user }} group={{ slave_user }} mode=0600 - name: install slave ssh public key - when: deployment_type != qadevel + when: buildslave_public_sshkey_file is defined copy: src={{ private }}/files/taskotron/{{ buildslave_public_sshkey_file }} dest={{ slave_home }}/.ssh/id_rsa.pub owner={{ slave_user }} group={{ slave_user }} mode=0644 - name: make sure master is in known_hosts - when: deployment_type != qadevel + when: buildslave_public_sshkey_file is defined lineinfile: dest=/home/{{ slave_user }}/.ssh/known_hosts regexp='{{ buildmaster }}' line='{{ buildmaster }} {{ buildmaster_pubkey }}' create=yes owner={{ slave_user }} group={{ slave_user }} - name: generate buildslave service file @@ -38,4 +38,3 @@ - name: start and enable buildslave service service: name=buildslave enabled=yes state=started - From 63217d371123f160a071c5223ec133dc27a2b78a Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 09:53:27 +0000 Subject: [PATCH 047/193] buildslave passwords for qadevel buildslaves might be useful --- .../buildslave-configure/templates/buildbot.tac.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 index d9837cc5cd..f7b805a0b3 100644 --- a/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 +++ b/roles/taskotron/buildslave-configure/templates/buildbot.tac.j2 @@ -39,6 +39,12 @@ passwd = '{{ stg_buildslave_password }}' {% if deployment_type == 'prod' %} passwd = '{{ prod_buildslave_password }}' {% endif %} +{% if deployment_type == 'qadevel-prod' %} +passwd = '{{ qadevel_prod_buildslave_password }}' +{% endif %} +{% if deployment_type == 'qadevel-stg' %} +passwd = '{{ qadevel_stg_buildslave_password }}' +{% endif %} keepalive = 600 usepty = 0 From afe1f6217db201bebcc92402ba79abfd90186ffd Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 18 Aug 2014 12:46:19 +0200 Subject: [PATCH 048/193] Move the pkgdb-sync-bugzilla.cron to pkgdb02 and ansible --- roles/pkgdb2/tasks/main.yml | 9 +++++++++ roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron | 4 ++++ 2 files changed, 13 insertions(+) create mode 100644 roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron diff --git a/roles/pkgdb2/tasks/main.yml b/roles/pkgdb2/tasks/main.yml index a01f76f99d..a67284185c 100644 --- a/roles/pkgdb2/tasks/main.yml +++ b/roles/pkgdb2/tasks/main.yml @@ -48,6 +48,15 @@ notify: - restart apache +- name: Install the pkgdb <-> bugzilla sync script + when: inventory_hostname.startswith('pkgdb02') + template: src={{ item.file }} + dest={{ item.location }}/{{ item.file }} + with_items: + - { file: 'pkgdb-sync-bugzilla.cron', location: /etc/cron.d } + tags: + - config + - name: set sebooleans so pkgdb2 can talk to the db action: seboolean name=httpd_can_network_connect_db state=true diff --git a/roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron b/roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron new file mode 100644 index 0000000000..43ea1c2a41 --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb-sync-bugzilla.cron @@ -0,0 +1,4 @@ +# Synchronize pkgdb with bugzilla +# +*/10 * * * * root PKGDB2_CONFIG=/etc/pkgdb2/pkgdb2.cfg /usr/bin/pkgdb-sync-bugzilla + From 59cae641298be3f288ae0250bdf3a7a5b89a16b5 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 18 Aug 2014 12:50:23 +0200 Subject: [PATCH 049/193] Adjust the pkgdb2 configuration file to include the settings for the cron job --- roles/pkgdb2/templates/pkgdb2.cfg | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg index f33dc777df..1f8d00bac8 100644 --- a/roles/pkgdb2/templates/pkgdb2.cfg +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -46,6 +46,16 @@ PKGDB2_BUGZILLA_USER = '{{ bugzilla_user }}' PKGDB2_BUGZILLA_PASSWORD = '{{ bugzilla_password }}' +# Settings specific to the ``pkgdb-sync-bugzilla`` script/cron +PKGDB2_BUGZILLA_NOTIFY_EMAIL = [ + 'kevin@fedoraproject.org', + 'pingou@fedoraproject'] +BUGZILLA_COMPONENT_API = "component.get" +PKGDB2_BUGZILLA_NOTIFY_USER = '{{ bugzilla_user }}' +PKGDB2_BUGZILLA_NOTIFY_PASSWORD = '{{ bugzilla_user }}' +PKGDB2_BUGZILLA_DRY_RUN = False + + ### FAS information ## URL to the FAS instance to query From d7f5bf91e1d30fb4e4fe868fbf5f997a966ae9ac Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 18 Aug 2014 11:10:38 +0000 Subject: [PATCH 050/193] buildmaster: add step status debug line in the email body --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index edab2832d2..5f492e3cda 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -252,11 +252,12 @@ def html_message_formatter(mode, name, build, results, master_status): break name = "%s.%s" % (log.getStep().getName(), log.getName()) + status, dummy = log.getStep().getResults() content = log.getText().splitlines() # Note: can be VERY LARGE url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), log.getStep().getName(), log.getName()) - + text.append(u'DEBUG: status=%s dummy=%s' % (status, dummy)) text.append(u'Detailed log of last build step: %s' % (url, url)) text.append(u'
') From 495f4c88ccf66a25611a0b2e8564e986df7a8870 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 18 Aug 2014 13:23:53 +0200 Subject: [PATCH 051/193] Fix variable name, password means password --- roles/pkgdb2/templates/pkgdb2.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg index 1f8d00bac8..ad2bbc9d87 100644 --- a/roles/pkgdb2/templates/pkgdb2.cfg +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -52,7 +52,7 @@ PKGDB2_BUGZILLA_NOTIFY_EMAIL = [ 'pingou@fedoraproject'] BUGZILLA_COMPONENT_API = "component.get" PKGDB2_BUGZILLA_NOTIFY_USER = '{{ bugzilla_user }}' -PKGDB2_BUGZILLA_NOTIFY_PASSWORD = '{{ bugzilla_user }}' +PKGDB2_BUGZILLA_NOTIFY_PASSWORD = '{{ bugzilla_password }}' PKGDB2_BUGZILLA_DRY_RUN = False From e220eceb762c7f40269b350c357e58e57f2512c2 Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Mon, 18 Aug 2014 12:22:48 +0000 Subject: [PATCH 052/193] adding locks to builders to eliminate concurrent builds on slaves --- .../templates/taskotron.master.cfg.j2 | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 5f492e3cda..645d53f32d 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -159,22 +159,40 @@ factory.addStep(ShellCommand(command=["runtask", '-i', # capture the taskotron log factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log")) + +####### RESOURCE LOCKS +# +# This is a set of resource locks to make sure that we don't have too many things +# going on on each slave at one time. + +from buildbot import locks + +build_lock = locks.SlaveLock("slave_builds", + maxCount = 1) + + + +####### Builders +# +# The builders associate factories with slaves which are capable of executing those factory's tasks + from buildbot.config import BuilderConfig + c['builders'] = [] c['builders'].append( BuilderConfig(name="x86_64", slavenames=[{% for buildslave in x86_64_buildslaves %}"{{ buildslave }}",{% endfor %}], - factory=factory)) + factory=factory, locks=[build_lock.access('counting')])) c['builders'].append( BuilderConfig(name="i386", slavenames=[{% for buildslave in i386_buildslaves %}"{{ buildslave }}",{% endfor %}], - factory=factory)) + factory=factory, locks=[build_lock.access('counting')])) c['builders'].append( BuilderConfig(name="all", slavenames=[{% for buildslave in buildslaves %}"{{ buildslave }}",{% endfor %}], - factory=factory)) + factory=factory, locks=[build_lock.access('counting')])) ####### STATUS TARGETS From 149a37bdd37b40b0e0964446d9880afd69e80ae4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 18 Aug 2014 12:24:32 +0000 Subject: [PATCH 053/193] buildmaster: add missing import --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 645d53f32d..5fecdd5ce0 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -233,6 +233,7 @@ from buildbot.status.mail import MailNotifier from buildbot.status.builder import Results import cgi +import re # http://docs.buildbot.net/current/manual/cfg-statustargets.html#mailnotifier def html_message_formatter(mode, name, build, results, master_status): From 881f4e3cdaa82fffe1f8757eb9b4fec413323be3 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Mon, 18 Aug 2014 14:26:52 +0200 Subject: [PATCH 054/193] Fix typo in email in the pkgdb2 config file --- roles/pkgdb2/templates/pkgdb2.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg index ad2bbc9d87..828d41d7cf 100644 --- a/roles/pkgdb2/templates/pkgdb2.cfg +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -49,7 +49,7 @@ PKGDB2_BUGZILLA_PASSWORD = '{{ bugzilla_password }}' # Settings specific to the ``pkgdb-sync-bugzilla`` script/cron PKGDB2_BUGZILLA_NOTIFY_EMAIL = [ 'kevin@fedoraproject.org', - 'pingou@fedoraproject'] + 'pingou@fedoraproject.org'] BUGZILLA_COMPONENT_API = "component.get" PKGDB2_BUGZILLA_NOTIFY_USER = '{{ bugzilla_user }}' PKGDB2_BUGZILLA_NOTIFY_PASSWORD = '{{ bugzilla_password }}' From 1db414cc9d03999149789089543f59506f50673c Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Mon, 18 Aug 2014 14:20:21 +0000 Subject: [PATCH 055/193] Fix deployment_type --- .../buildmaster-configure/templates/ci.master.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 index e2ec5ac8a6..f6fc887d64 100644 --- a/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/ci.master.cfg.j2 @@ -178,10 +178,10 @@ from buildbot.status.web import authz, auth authz_cfg=authz.Authz( # change any of these to True to enable; see the manual for more # options - {% if deployment_type == 'stg' %} + {% if deployment_type == 'qadevel-stg' %} auth=auth.BasicAuth([("{{ qadevel_stg_buildbot_master_user }}","{{ qadevel_stg_buildbot_master_password }}")]), {% endif %} - {% if deployment_type == 'prod' %} + {% if deployment_type == 'qadevel-prod' %} auth=auth.BasicAuth([("{{ qadevel_prod_buildbot_master_user }}","{{ qadevel_prod_buildbot_master_password }}")]), {% endif %} gracefulShutdown = False, From 03f908c8ce126604852ea753d5a65685443e2d5e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 15:11:32 +0000 Subject: [PATCH 056/193] Point memcached0{1,2} at the right place. --- roles/hosts/files/packages03.stg.phx2.fedoraproject.org-hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hosts/files/packages03.stg.phx2.fedoraproject.org-hosts b/roles/hosts/files/packages03.stg.phx2.fedoraproject.org-hosts index 89d276b6b3..cc6fa1c424 100644 --- a/roles/hosts/files/packages03.stg.phx2.fedoraproject.org-hosts +++ b/roles/hosts/files/packages03.stg.phx2.fedoraproject.org-hosts @@ -6,7 +6,7 @@ 10.5.126.88 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 fedoraproject.org 10.5.126.83 pkgs.fedoraproject.org pkgs pkgs01 10.5.126.81 app01.phx2.fedoraproject.org app1 app3 app5 bapp1 app01 app03 app05 bapp01 bapp02 -10.5.126.81 memcached04.phx2.fedoraproject.org memcached04 memcached03 +10.5.126.81 memcached04.phx2.fedoraproject.org memcached04 memcached03 memcached01 memcached02 10.5.126.91 value03.phx2.fedoraproject.org value3 value03 10.5.125.119 nfs01.phx2.fedoraproject.org nfs01 nfs1 10.5.126.92 noc01.phx2.fedoraproject.org noc1 noc01 From b547198a95f6f732427ef6506b230ff5f7d62b4b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 15:52:05 +0000 Subject: [PATCH 057/193] Ban staging from prod. --- roles/base/templates/iptables/iptables | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 97160bdbce..7b995743ab 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -30,6 +30,12 @@ -A INPUT -p tcp -m tcp --dport 5666 -s 209.132.181.35 -j ACCEPT -A INPUT -p tcp -m tcp --dport 5666 -s 10.5.126.41 -j ACCEPT +# ban staging from non-staging hosts +{% if env != 'staging' %} +{% for host in groups['staging'] %} +-A INPUT -s {{ hostvars[host]['eth0_ip'] }} -j REJECT --reject-with icmp-host-prohibited +{% endfor %} +{% endif %} # if the host/group defines incoming tcp_ports - allow them {% if tcp_ports is defined %} From 056d0f22f9d4b0e9cfd1e50b3312db5be0b73f00 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 16:05:44 +0000 Subject: [PATCH 058/193] The Internet seems to think I should do this. --- roles/base/templates/iptables/iptables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 7b995743ab..6f4752ae64 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -33,7 +33,7 @@ # ban staging from non-staging hosts {% if env != 'staging' %} {% for host in groups['staging'] %} --A INPUT -s {{ hostvars[host]['eth0_ip'] }} -j REJECT --reject-with icmp-host-prohibited +-A INPUT -s {{ hostvars[host]['ansible_eth0']['ipv4']['address'] }} -j REJECT --reject-with icmp-host-prohibited {% endfor %} {% endif %} From d530e7da490c0330571d4a8c5df3a08cfe6d2698 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 16:08:56 +0000 Subject: [PATCH 059/193] Some comments. --- roles/base/templates/iptables/iptables | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 6f4752ae64..63aa1d2905 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -33,7 +33,10 @@ # ban staging from non-staging hosts {% if env != 'staging' %} {% for host in groups['staging'] %} +{% if 'eth0_ip' in hostvars[host] %}# {{ host }} -A INPUT -s {{ hostvars[host]['ansible_eth0']['ipv4']['address'] }} -j REJECT --reject-with icmp-host-prohibited +{% else %}# {{ host }} has no 'eth0_ip' listed +{% endif %} {% endfor %} {% endif %} From 20344c943b2d4b49609cb63a20bdfcd8707d90ec Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 16:09:48 +0000 Subject: [PATCH 060/193] Back to our explicit IP. --- roles/base/templates/iptables/iptables | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/base/templates/iptables/iptables b/roles/base/templates/iptables/iptables index 63aa1d2905..f67fa63353 100644 --- a/roles/base/templates/iptables/iptables +++ b/roles/base/templates/iptables/iptables @@ -34,7 +34,7 @@ {% if env != 'staging' %} {% for host in groups['staging'] %} {% if 'eth0_ip' in hostvars[host] %}# {{ host }} --A INPUT -s {{ hostvars[host]['ansible_eth0']['ipv4']['address'] }} -j REJECT --reject-with icmp-host-prohibited +-A INPUT -s {{ hostvars[host]['eth0_ip'] }} -j REJECT --reject-with icmp-host-prohibited {% else %}# {{ host }} has no 'eth0_ip' listed {% endif %} {% endfor %} From f797aad45c63316d549e42546173bf0aceaafa9d Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Mon, 18 Aug 2014 17:36:07 +0000 Subject: [PATCH 061/193] allow traffic on virbr0 --- roles/base/templates/iptables/iptables.kojibuilder | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/base/templates/iptables/iptables.kojibuilder b/roles/base/templates/iptables/iptables.kojibuilder index 1d8e865fa1..439db80b53 100644 --- a/roles/base/templates/iptables/iptables.kojibuilder +++ b/roles/base/templates/iptables/iptables.kojibuilder @@ -7,6 +7,8 @@ # loopback allowed -A INPUT -i lo -j ACCEPT -A OUTPUT -o lo -j ACCEPT +-A INPUT -i virbr0 -j ACCEPT +-A OUTPUT -o virbr0 -j ACCEPT -A INPUT -d 127.0.0.0/8 -j ACCEPT -A OUTPUT -d 127.0.0.0/8 -j ACCEPT From f2b055dbbfe0f190516abae2ed7020211b9005a2 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Mon, 18 Aug 2014 19:15:49 +0000 Subject: [PATCH 062/193] update the dhpcd for retrace boxes --- .../dhcpd.conf.noc01.phx2.fedoraproject.org | 184 +++++++++--------- 1 file changed, 91 insertions(+), 93 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 563d313c3f..0624ece83e 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -1423,21 +1423,12 @@ shared-network qa { next-server 10.5.126.41; filename "pxelinux.0"; - host retrace03 { - hardware ethernet ec:f4:bb:c0:38:a0; - fixed-address 10.5.124.230; - option host-name "retrace03.qa.fedoraproject.org"; - next-server 10.5.126.41; - filename "uefi/shim.efi"; + host virthost-comm01 { + hardware ethernet 5c:f3:fc:4d:db:fc; + fixed-address 10.5.124.131; + option host-name "virthost-comm01"; } - - host virthost-comm01 { - hardware ethernet 5c:f3:fc:4d:db:fc; - fixed-address 10.5.124.131; - option host-name "virthost-comm01"; - } - # # All staticly assigned, but listed here for completeness. # @@ -1448,102 +1439,101 @@ shared-network qa { # 5C:F3:FC:85:64:38 - ppc-builder4.qa.fedoraproject.org - 10.5.124.216 # 5C:F3:FC:85:64:35 - ppc-composer.qa.fedoraproject.org - 10.5.124.217 - host ppc-comm01-mgmt { - hardware ethernet 5c:f3:fc:2e:93:72; - fixed-address 10.5.124.231; - option host-name "ppc-comm01-mgmt"; - } + host ppc-comm01-mgmt { + hardware ethernet 5c:f3:fc:2e:93:72; + fixed-address 10.5.124.231; + option host-name "ppc-comm01-mgmt"; + } - host ppc-comm01 { - hardware ethernet 6c:ae:8b:00:0f:f0; - fixed-address 10.5.124.219; - option host-name "ppc-comm01"; - filename "yaboot"; - next-server 10.5.126.41; - } + host ppc-comm01 { + hardware ethernet 6c:ae:8b:00:0f:f0; + fixed-address 10.5.124.219; + option host-name "ppc-comm01"; + filename "yaboot"; + } - host qa01 { - hardware ethernet 00:21:5E:C7:5C:84; - fixed-address 10.5.124.151; - } + host qa01 { + hardware ethernet 00:21:5E:C7:5C:84; + fixed-address 10.5.124.151; + } - host qa02 { - hardware ethernet 00:21:5e:c6:cc:9c; - fixed-address 10.5.124.152; - } + host qa02 { + hardware ethernet 00:21:5e:c6:cc:9c; + fixed-address 10.5.124.152; + } - host qa03 { - hardware ethernet 00:21:5E:C6:CD:48; - fixed-address 10.5.124.153; - } + host qa03 { + hardware ethernet 00:21:5E:C6:CD:48; + fixed-address 10.5.124.153; + } - host qa04 { - hardware ethernet 00:21:5E:C7:2A:1C; - fixed-address 10.5.124.154; - } + host qa04 { + hardware ethernet 00:21:5E:C7:2A:1C; + fixed-address 10.5.124.154; + } - host qa05 { - hardware ethernet 00:21:5E:C7:5F:04; - fixed-address 10.5.124.155; - } + host qa05 { + hardware ethernet 00:21:5E:C7:5F:04; + fixed-address 10.5.124.155; + } - host qa06 { - hardware ethernet 00:21:5E:C6:57:08; - fixed-address 10.5.124.156; - } + host qa06 { + hardware ethernet 00:21:5E:C6:57:08; + fixed-address 10.5.124.156; + } - host qa07 { - hardware ethernet E4:1F:13:E5:53:94; - fixed-address 10.5.124.157; - } + host qa07 { + hardware ethernet E4:1F:13:E5:53:94; + fixed-address 10.5.124.157; + } - host qa08 { - hardware ethernet E4:1F:13:E5:46:80; - fixed-address 10.5.124.158; - } + host qa08 { + hardware ethernet E4:1F:13:E5:46:80; + fixed-address 10.5.124.158; + } - host virt01 { - hardware ethernet 52:54:00:a2:de:30; - fixed-address 10.5.124.159; - option host-name "virt01"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt01 { + hardware ethernet 52:54:00:a2:de:30; + fixed-address 10.5.124.159; + option host-name "virt01"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } - host virt02 { - hardware ethernet 52:54:00:fe:22:ff; - fixed-address 10.5.124.160; - option host-name "virt02"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt02 { + hardware ethernet 52:54:00:fe:22:ff; + fixed-address 10.5.124.160; + option host-name "virt02"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } - host virt03 { - hardware ethernet 52:54:00:c5:04:14; - fixed-address 10.5.124.161; - option host-name "virt03"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt03 { + hardware ethernet 52:54:00:c5:04:14; + fixed-address 10.5.124.161; + option host-name "virt03"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } - host virt04 { - hardware ethernet 52:54:00:b5:97:30; - fixed-address 10.5.124.162; - option host-name "virt04"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt04 { + hardware ethernet 52:54:00:b5:97:30; + fixed-address 10.5.124.162; + option host-name "virt04"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } - host virt05 { - hardware ethernet 52:54:00:9a:25:d3; - fixed-address 10.5.124.163; - option host-name "virt05"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt05 { + hardware ethernet 52:54:00:9a:25:d3; + fixed-address 10.5.124.163; + option host-name "virt05"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } - host virt06 { - hardware ethernet 52:54:00:78:ae:44; - fixed-address 10.5.124.164; - option host-name "virt06"; - option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; - } + host virt06 { + hardware ethernet 52:54:00:78:ae:44; + fixed-address 10.5.124.164; + option host-name "virt06"; + option ntp-servers 66.187.233.4, 192.43.244.18, 128.118.25.5, 204.152.184.72; + } host virt07 { hardware ethernet 52:54:00:00:eb:e9; @@ -1753,6 +1743,14 @@ shared-network qa { next-server 10.5.126.41; filename "pxelinux.0"; + host retrace03 { + hardware ethernet ec:f4:bb:c0:38:a0; + fixed-address 10.5.131.13; + option host-name "retrace03.qa.fedoraproject.org"; + next-server 10.5.126.41; + filename "pxelinux.0"; + } + host retrace04 { hardware ethernet ec:f4:bb:c0:46:38; fixed-address 10.5.131.14; From b229169df5c3a12e4ea4e29796cf4f7678ed894c Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 19:32:48 +0000 Subject: [PATCH 063/193] Convert jenkins over to use tcp_ports vars. --- inventory/host_vars/jenkins.cloud.fedoraproject.org | 1 + playbooks/groups/jenkins-cloud.yml | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/inventory/host_vars/jenkins.cloud.fedoraproject.org b/inventory/host_vars/jenkins.cloud.fedoraproject.org index df29f7da60..4f98ae70c3 100644 --- a/inventory/host_vars/jenkins.cloud.fedoraproject.org +++ b/inventory/host_vars/jenkins.cloud.fedoraproject.org @@ -6,6 +6,7 @@ security_group: jenkins zone: nova hostbase: jenkins-master- public_ip: 209.132.184.153 +tcp_ports: [22, 80, 443] root_auth_users: pingou description: jenkins cloud master volumes: ['-d /dev/vdb vol-00000011', '-d /dev/vdc vol-0000002b'] diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index ed318f2bff..ded97c2e5f 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -39,13 +39,6 @@ - name: mount our persistent space action: mount name=/var/lib/jenkins src='LABEL=jenkins' fstype=ext4 state=mounted - - name: poke firewall holes - action: command lokkit {{ item }} - with_items: - - --service=ssh - - --service=https - - --service=http - - name: install pkgs for jenkins action: yum state=installed pkg={{ item }} with_items: From 605cb0f0e2d0de6a50ad86ce31fb9c07a91dc05b Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 19:53:32 +0000 Subject: [PATCH 064/193] Make it a group_var. --- inventory/group_vars/jenkins-cloud | 2 ++ inventory/host_vars/jenkins.cloud.fedoraproject.org | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/jenkins-cloud b/inventory/group_vars/jenkins-cloud index bbefeaf0a9..964d8868db 100644 --- a/inventory/group_vars/jenkins-cloud +++ b/inventory/group_vars/jenkins-cloud @@ -1,5 +1,7 @@ postfix_group: jenkins-cloud +tcp_ports: [22, 80, 443] + # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: - service: shell diff --git a/inventory/host_vars/jenkins.cloud.fedoraproject.org b/inventory/host_vars/jenkins.cloud.fedoraproject.org index 4f98ae70c3..df29f7da60 100644 --- a/inventory/host_vars/jenkins.cloud.fedoraproject.org +++ b/inventory/host_vars/jenkins.cloud.fedoraproject.org @@ -6,7 +6,6 @@ security_group: jenkins zone: nova hostbase: jenkins-master- public_ip: 209.132.184.153 -tcp_ports: [22, 80, 443] root_auth_users: pingou description: jenkins cloud master volumes: ['-d /dev/vdb vol-00000011', '-d /dev/vdc vol-0000002b'] From 0fdbdda117d7195653821e5feaded5ee7a4d5b13 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Mon, 18 Aug 2014 20:03:46 +0000 Subject: [PATCH 065/193] Also convert copr from lokkit to `tcp_ports` --- inventory/host_vars/209.132.184.144 | 1 + inventory/host_vars/copr-fe-dev.cloud.fedoraproject.org | 2 +- playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml | 8 -------- playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml | 8 -------- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/inventory/host_vars/209.132.184.144 b/inventory/host_vars/209.132.184.144 index b45f61445f..3fc35f8f48 100644 --- a/inventory/host_vars/209.132.184.144 +++ b/inventory/host_vars/209.132.184.144 @@ -9,3 +9,4 @@ public_ip: 209.132.184.144 root_auth_users: bkabrda ryanlerch pingou msuchy sgallagh nb asamalik description: copr frontend server volumes: ['-d /dev/vdb vol-0000000f'] +tcp_ports: [22, 80, 443] diff --git a/inventory/host_vars/copr-fe-dev.cloud.fedoraproject.org b/inventory/host_vars/copr-fe-dev.cloud.fedoraproject.org index a191d537ee..b5e8f7a84a 100644 --- a/inventory/host_vars/copr-fe-dev.cloud.fedoraproject.org +++ b/inventory/host_vars/copr-fe-dev.cloud.fedoraproject.org @@ -8,4 +8,4 @@ hostbase: copr-fe-dev- public_ip: 209.132.184.183 root_auth_users: bkabrda ryanlerch pingou msuchy tradej asamalik description: copr frontend server - dev instance -tcp_ports: ['22', '80', '443'] +tcp_ports: [22, 80, 443] diff --git a/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml index 970e56b914..9ccb6f01e3 100644 --- a/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-fe-dev.cloud.fedoraproject.org.yml @@ -67,14 +67,6 @@ - name: copy pg_hba.conf action: copy src="{{ files }}/copr/fe/pg/pg_hba.conf" dest=/var/lib/pgsql/data/pg_hba.conf owner=postgres group=postgres mode=0600 - # open up ports (22, 80, 443) - - name: poke holes in the firewall - action: command lokkit {{ item }} - with_items: - - --service=ssh - - --service=https - - --service=http - - name: copy httpd ssl certificates (crt) action: copy src="{{ puppet_private }}/httpd/copr-fe.fedoraproject.org.crt" dest="/etc/pki/tls/certs/" owner=root group=root mode=0600 tags: diff --git a/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml b/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml index 4ea95482d0..cf58873263 100644 --- a/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/copr-fe.cloud.fedoraproject.org.yml @@ -73,14 +73,6 @@ - name: copy pg_hba.conf action: copy src="{{ files }}/copr/fe/pg/pg_hba.conf" dest=/var/lib/pgsql/data/pg_hba.conf owner=postgres group=postgres mode=0600 - # open up ports (22, 80, 443) - - name: poke holes in the firewall - action: command lokkit {{ item }} - with_items: - - --service=ssh - - --service=https - - --service=http - - name: copy httpd ssl certificates (crt) action: copy src="{{ puppet_private }}/httpd/copr-fe.fedoraproject.org.crt" dest="/etc/pki/tls/certs/" owner=root group=root mode=0600 tags: From 4ae9276aca1c4d35ce89b404f56f0ecb491ab12a Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 19 Aug 2014 12:47:10 +0000 Subject: [PATCH 066/193] buildmaster: put log of all failed steps into email reports --- .../templates/taskotron.master.cfg.j2 | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 5fecdd5ce0..e57cfd075d 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -231,6 +231,7 @@ c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) from buildbot.status.mail import MailNotifier from buildbot.status.builder import Results +from buildbot.status.results import FAILURE, EXCEPTION, WARNINGS import cgi import re @@ -244,7 +245,7 @@ def html_message_formatter(mode, name, build, results, master_status): """ result = Results[results] - limit_lines = 80 + limit_lines = 40 text = list() text.append(u'

Build status: %s

' % result.upper()) text.append(u'') @@ -256,41 +257,47 @@ def html_message_formatter(mode, name, build, results, master_status): ) text.append(u'' % build.getReason()) text.append(u'
Build Reason:%s
') - text.append(u'
') logs = [] steps = build.getSteps() for step in steps: - if step.getName() == 'runtask': - logs = step.getLogs() - break + status, dummy = step.getResults() + + if status not in [FAILURE, EXCEPTION, WARNINGS]: + continue + + logs = step.getLogs() + + # logs within a step are in reverse order. Search back until we find stdio + for log in reversed(logs): + if log.getName() == 'stdio': + break + + name = "%s.%s" % (step).getName(), log.getName()) + status, dummy = log.getStep().getResults() + content = log.getText().splitlines() # Note: can be VERY LARGE + url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), + step.getName(), log.getName()) + + text.append(u'

') + text.append(u'Detailed log of the %s build step: %s' + % (name, url, url)) + text.append(u'
') + text.append(u'

Last %d lines of "%s"

' % (limit_lines, name)) + + unilist = list() + for line in content[len(content)-limit_lines:]: + unilist.append(cgi.escape(unicode(line,'utf-8'))) + text.append(u'
')
+            text.extend(unilist)
+            text.append(u'
') - # logs within a step are in reverse order. Search back until we find stdio - for log in reversed(logs): - if log.getName() == 'stdio': - break - name = "%s.%s" % (log.getStep().getName(), log.getName()) - status, dummy = log.getStep().getResults() - content = log.getText().splitlines() # Note: can be VERY LARGE - url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), - log.getStep().getName(), - log.getName()) - text.append(u'DEBUG: status=%s dummy=%s' % (status, dummy)) - text.append(u'Detailed log of last build step: %s' - % (url, url)) - text.append(u'
') - text.append(u'

Last %d lines of "%s"

' % (limit_lines, name)) - unilist = list() - for line in content[len(content)-limit_lines:]: - unilist.append(cgi.escape(unicode(line,'utf-8'))) - text.append(u'
')
-        text.extend(unilist)
-        text.append(u'
') text.append(u'

') text.append(u'-The Buildbot') taskname = re.search("task=\['(.*)\.yml'\]", t).group(1) test.insert(0, '

Task name: %s

' % taskname) + return { 'body': u"\n".join(text), 'type': 'html' From 59bbd16fcaf35b3893a923ed9d72f9819c74b1e3 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 19 Aug 2014 12:54:20 +0000 Subject: [PATCH 067/193] Fix typo --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index e57cfd075d..dbddad051e 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -273,7 +273,7 @@ def html_message_formatter(mode, name, build, results, master_status): if log.getName() == 'stdio': break - name = "%s.%s" % (step).getName(), log.getName()) + name = "%s.%s" % (step.getName(), log.getName()) status, dummy = log.getStep().getResults() content = log.getText().splitlines() # Note: can be VERY LARGE url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), From 24702b7569cc4a3e02b328003f6c2f3e10375a96 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Tue, 19 Aug 2014 14:18:47 +0000 Subject: [PATCH 068/193] Send fetch_activity's output via mail --- .../taskotron-trigger/templates/fetch_activity.cron.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/taskotron/taskotron-trigger/templates/fetch_activity.cron.j2 b/roles/taskotron/taskotron-trigger/templates/fetch_activity.cron.j2 index 4cbcfd7061..cbae16ff52 100644 --- a/roles/taskotron/taskotron-trigger/templates/fetch_activity.cron.j2 +++ b/roles/taskotron/taskotron-trigger/templates/fetch_activity.cron.j2 @@ -1,3 +1,4 @@ MAILFROM=taskotron@fedoraproject.org MAILTO=sysadmin-qa-members@fedoraproject.org -0 1,13 * * * root python /root/fetch_activity.py -u {{ resultsdb_url }} -f {{ resultsdb_frontend_url }} -t 12 > /root/fetch_activity_logs/fetch_activity_$(date "+\%H\%M_\%m\%d\%y").log +#0 1,13 * * * root python /root/fetch_activity.py -u {{ resultsdb_url }} -f {{ resultsdb_frontend_url }} -t 12 > /root/fetch_activity_logs/fetch_activity_$(date "+\%H\%M_\%m\%d\%y").log +0 1,13 * * * root python /root/fetch_activity.py -u {{ resultsdb_url }} -f {{ resultsdb_frontend_url }} -t 12 From 92d3d822baf5106b65ff50c263390997875d5736 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 19 Aug 2014 14:40:26 +0000 Subject: [PATCH 069/193] Try monitoring memcached proc with nagios. --- .../files/nagios/services/memcached.cfg | 12 ++++++++++++ roles/nagios_server/files/nrpe.cfg | 1 + 2 files changed, 13 insertions(+) create mode 100644 roles/nagios_server/files/nagios/services/memcached.cfg diff --git a/roles/nagios_server/files/nagios/services/memcached.cfg b/roles/nagios_server/files/nagios/services/memcached.cfg new file mode 100644 index 0000000000..9f497b50c6 --- /dev/null +++ b/roles/nagios_server/files/nagios/services/memcached.cfg @@ -0,0 +1,12 @@ +define service { + host_name memcached01 + service_description Check memcached daemon + check_command check_by_nrpe!check_memcache + use defaulttemplate +} +define service { + host_name memcached02 + service_description Check memcached daemon + check_command check_by_nrpe!check_memcache + use defaulttemplate +} diff --git a/roles/nagios_server/files/nrpe.cfg b/roles/nagios_server/files/nrpe.cfg index 65a64aac91..4a263fa926 100644 --- a/roles/nagios_server/files/nrpe.cfg +++ b/roles/nagios_server/files/nrpe.cfg @@ -237,6 +237,7 @@ command[check_fcomm_cache_worker_proc]=/usr/lib64/nagios/plugins/check_procs -c command[check_fcomm_queue]=/usr/lib64/nagios/plugins/check_fcomm_queue command[check_redis_proc]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -C 'redis-server' -u redis command[check_openvpn_link]=/usr/lib64/nagios/plugins/check_ping -H 192.168.1.58 -w 375.0,20% -c 500,60% +command[check_memcache]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -a '/usr/bin/memcached' -u memcached # The following are fedmsg/datanommer checks to be run on busgateway01. # They check for the time since the latest message in any particular category. From 7893c7e2a1d4bfffb07ec9c2f0546a7ce38814cd Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 19 Aug 2014 14:46:03 +0000 Subject: [PATCH 070/193] Have to add memcache definition here too. --- roles/nagios_client/tasks/main.yml | 1 + roles/nagios_client/templates/check_memcache_proc.cfg.j2 | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 roles/nagios_client/templates/check_memcache_proc.cfg.j2 diff --git a/roles/nagios_client/tasks/main.yml b/roles/nagios_client/tasks/main.yml index 5a62458755..231e03c7bd 100644 --- a/roles/nagios_client/tasks/main.yml +++ b/roles/nagios_client/tasks/main.yml @@ -90,6 +90,7 @@ - check_fedmsg_consumers.cfg - check_supybot_fedmsg_plugin.cfg - check_datanommer_history.cfg + - check_memcache.cfg notify: - restart nrpe tags: diff --git a/roles/nagios_client/templates/check_memcache_proc.cfg.j2 b/roles/nagios_client/templates/check_memcache_proc.cfg.j2 new file mode 100644 index 0000000000..b350a654e3 --- /dev/null +++ b/roles/nagios_client/templates/check_memcache_proc.cfg.j2 @@ -0,0 +1,2 @@ +command[check_memcache]=/usr/lib64/nagios/plugins/check_procs -c 1:1 -a '/usr/bin/memcached' -u memcached + From bfddc11aba6d64d1cc91ad653e9f981db76fe116 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 19 Aug 2014 14:47:41 +0000 Subject: [PATCH 071/193] Rename. --- .../{check_memcache_proc.cfg.j2 => check_memcache.cfg.j2} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/nagios_client/templates/{check_memcache_proc.cfg.j2 => check_memcache.cfg.j2} (100%) diff --git a/roles/nagios_client/templates/check_memcache_proc.cfg.j2 b/roles/nagios_client/templates/check_memcache.cfg.j2 similarity index 100% rename from roles/nagios_client/templates/check_memcache_proc.cfg.j2 rename to roles/nagios_client/templates/check_memcache.cfg.j2 From a01469ad7401b808c3307d8978eea9824d75cb5d Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 15:01:48 +0000 Subject: [PATCH 072/193] Set memcached to always restart. --- handlers/restart_services.yml | 3 +++ roles/memcached/files/memcached-systemdoverride.conf | 2 ++ roles/memcached/tasks/main.yml | 12 ++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 roles/memcached/files/memcached-systemdoverride.conf diff --git a/handlers/restart_services.yml b/handlers/restart_services.yml index 41094e7478..f1664f2d93 100644 --- a/handlers/restart_services.yml +++ b/handlers/restart_services.yml @@ -121,3 +121,6 @@ - name: restart memcached service: name=memcached state=restarted + +- name: reload systemd + action: systemctl daemon-reload diff --git a/roles/memcached/files/memcached-systemdoverride.conf b/roles/memcached/files/memcached-systemdoverride.conf new file mode 100644 index 0000000000..8a764e3478 --- /dev/null +++ b/roles/memcached/files/memcached-systemdoverride.conf @@ -0,0 +1,2 @@ +[Service] +Restart=always diff --git a/roles/memcached/tasks/main.yml b/roles/memcached/tasks/main.yml index a75007853c..5061107152 100644 --- a/roles/memcached/tasks/main.yml +++ b/roles/memcached/tasks/main.yml @@ -19,3 +19,15 @@ - service - config - memcached + +- name: make systemd override dir + file: state=directory path=/etc/systemd/system/memcached.d mode=0755 owner=root group=root + tags: + - memcached + +- name: make systemd override config + copy: src=memcached-systemdoverride.conf dest=/etc/systemd/system/memcached.d/ + tags: + - memcached + notify: + - reload systemd From 28fbe5f39f45e1ec9da514cd7a7ee3d0229dd126 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 15:07:30 +0000 Subject: [PATCH 073/193] This is a command, not a systemctl module. --- handlers/restart_services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/restart_services.yml b/handlers/restart_services.yml index f1664f2d93..ac838050d8 100644 --- a/handlers/restart_services.yml +++ b/handlers/restart_services.yml @@ -123,4 +123,4 @@ service: name=memcached state=restarted - name: reload systemd - action: systemctl daemon-reload + command: systemctl daemon-reload From bd716d890d60a5ef65949d552406d75be46a33c6 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 15:08:15 +0000 Subject: [PATCH 074/193] Add a whitespace change to trigger the handler. --- roles/memcached/files/memcached-systemdoverride.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/memcached/files/memcached-systemdoverride.conf b/roles/memcached/files/memcached-systemdoverride.conf index 8a764e3478..98d34df46c 100644 --- a/roles/memcached/files/memcached-systemdoverride.conf +++ b/roles/memcached/files/memcached-systemdoverride.conf @@ -1,2 +1,3 @@ [Service] Restart=always + From 13d4e263d31369dc125af48d6eb6383278402081 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 15:56:55 +0000 Subject: [PATCH 075/193] Clean up retrace playbook, make it general for arm/non arm. --- inventory/inventory | 4 +++- playbooks/groups/{arm-retrace.yml => retrace.yml} | 7 ++----- 2 files changed, 5 insertions(+), 6 deletions(-) rename playbooks/groups/{arm-retrace.yml => retrace.yml} (79%) diff --git a/inventory/inventory b/inventory/inventory index cb5f76893b..5d97721832 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -25,8 +25,10 @@ arm03-qa01.cloud.fedoraproject.org arm01-releng00.arm.fedoraproject.org compose-x86-02.phx2.fedoraproject.org -[arm-retrace] +[retrace] arm01-retrace01.arm.fedoraproject.org +retrace03.qa.fedoraproject.org +retrace04.qa.fedoraproject.org [app-stg] app01.stg.phx2.fedoraproject.org diff --git a/playbooks/groups/arm-retrace.yml b/playbooks/groups/retrace.yml similarity index 79% rename from playbooks/groups/arm-retrace.yml rename to playbooks/groups/retrace.yml index 29617a4af1..854bc0c898 100644 --- a/playbooks/groups/arm-retrace.yml +++ b/playbooks/groups/retrace.yml @@ -1,10 +1,8 @@ -- name: Setup arm-retrace hosts - hosts: arm-retrace +- name: Setup retrace hosts + hosts: retrace user: root gather_facts: True - tags: - - arm-retrace vars_files: - /srv/web/infra/ansible/vars/global.yml @@ -21,7 +19,6 @@ - sudo tasks: - # this is how you include other task lists - include: "{{ tasks }}/2fa_client.yml" - include: "{{ tasks }}/motd.yml" - include: "{{ tasks }}/common_scripts.yml" From 11b9637d6bd50c2de3873f544ba59e54d8bcfd70 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 16:06:55 +0000 Subject: [PATCH 076/193] Some more arm-retrace/retrace cleanup. --- inventory/group_vars/{arm-retrace => retrace} | 4 ---- inventory/host_vars/arm01-retrace01.arm.fedoraproject.org | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) rename inventory/group_vars/{arm-retrace => retrace} (75%) create mode 100644 inventory/host_vars/arm01-retrace01.arm.fedoraproject.org diff --git a/inventory/group_vars/arm-retrace b/inventory/group_vars/retrace similarity index 75% rename from inventory/group_vars/arm-retrace rename to inventory/group_vars/retrace index e2a2e88f9b..a60e0563b2 100644 --- a/inventory/group_vars/arm-retrace +++ b/inventory/group_vars/retrace @@ -1,10 +1,6 @@ --- fas_client_groups: retrace freezes: false -# -# These are 32bit -# -libdir: /usr/lib sudoers: "{{ private }}/files/sudo/arm-retrace-sudoers" tcp_ports: [ 80 ] diff --git a/inventory/host_vars/arm01-retrace01.arm.fedoraproject.org b/inventory/host_vars/arm01-retrace01.arm.fedoraproject.org new file mode 100644 index 0000000000..897dd481a0 --- /dev/null +++ b/inventory/host_vars/arm01-retrace01.arm.fedoraproject.org @@ -0,0 +1,3 @@ +--- +# This is a 32bit host +libdir: /usr/lib From b532d53385cdeff5a63bc493e9abb430f6044abe Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 19 Aug 2014 16:28:39 +0000 Subject: [PATCH 077/193] No denyhosts for 7 --- playbooks/groups/retrace.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/retrace.yml b/playbooks/groups/retrace.yml index 854bc0c898..6da12268a6 100644 --- a/playbooks/groups/retrace.yml +++ b/playbooks/groups/retrace.yml @@ -14,7 +14,7 @@ - hosts - fas_client - rkhunter - - denyhosts + - { role: denyhosts, when: ansible_distribution_major_version != '7' } - nagios_client - sudo From 6857480956b66bf06fa6bd65b4e7355e8b3d5cd2 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Tue, 19 Aug 2014 20:15:53 +0200 Subject: [PATCH 078/193] Jenkins need to have nrpe installed to run the fedmsg/base role correctly --- playbooks/groups/jenkins-cloud.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index ded97c2e5f..2ac68408de 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -49,6 +49,7 @@ - httpd - openssh-clients - git + - nrpe tags: - packages From 7392fb99f6e8cb66c997969ead850e7e7c8b3e3b Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Tue, 19 Aug 2014 22:09:30 +0200 Subject: [PATCH 079/193] One package is F19+ only --- playbooks/groups/jenkins-cloud.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 2ac68408de..7e98a7f30c 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -333,6 +333,12 @@ tags: - packages + - name: install pkgs for jenkins for fedora systems > F19 + action: yum state=installed pkg={{ item }} + when: is_fedora is defined and ansible_distribution_major_version > 19 + with_items: + - sbt-extras + - name: install pkgs for jenkins for fedora systems action: yum state=installed pkg={{ item }} when: is_fedora is defined @@ -340,7 +346,6 @@ - python3 - python-nose-cover3 - python3-nose-cover3 - - sbt-extras - glibc.i686 - glibc-devel.i686 - libstdc++.i686 From 4c523b30a95ed464dd8b920876e53a5136504460 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Tue, 19 Aug 2014 20:24:51 +0000 Subject: [PATCH 080/193] Specify the password of the account when creating the FAS client --- scripts/auth-keys-from-fas | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auth-keys-from-fas b/scripts/auth-keys-from-fas index b288cc477c..9ec9577044 100755 --- a/scripts/auth-keys-from-fas +++ b/scripts/auth-keys-from-fas @@ -186,7 +186,7 @@ if __name__ == '__main__': conf['username'] = username conf['password'] = password - fas = AccountSystem(username=conf['username']) + fas = AccountSystem(username=conf['username'], password=conf['password']) from_string = '' if conf['limit_from']: From 8b673d0dbca51671fe99a09a221886ee8dca542a Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 07:57:12 +0200 Subject: [PATCH 081/193] Try F20+ --- playbooks/groups/jenkins-cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 7e98a7f30c..d92dd869e7 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -335,7 +335,7 @@ - name: install pkgs for jenkins for fedora systems > F19 action: yum state=installed pkg={{ item }} - when: is_fedora is defined and ansible_distribution_major_version > 19 + when: is_fedora is defined and ansible_distribution_major_version > 20 with_items: - sbt-extras From 291dab12e3885af6247c9d991af3f9f6edca5531 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 09:35:44 +0200 Subject: [PATCH 082/193] Update jenkins' plugins --- playbooks/groups/jenkins-cloud.yml | 46 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index d92dd869e7..7943ef0309 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -106,35 +106,35 @@ version: 0.5 sha: bd9df0507008255ad2ed046368d10a4d039a6cbcfefb53c71c1768cc0dcbf65b - name: cobertura - version: 1.9.3 - sha: 3db93d70486b80a904a74ce40b0ac6a7812d1f522f820d0e5d7b538401bc2946 + version: 1.9.5 + sha: a76bc1524efc5ba05672638001c0e951edd2a853d222efcfb035e02169e4252a - name: cvs - version: 2.11 - sha: 7c917bc824019a81d54472c525e4d724dfb4ae10b59bf64e692a2fc59fcd33cc + version: 2.12 + sha: 6e6dfd35e8501bf5f84a9d43d210db61165ce51a606327fc81f2efc5208478ba - name: external-monitor-job version: 1.2 sha: 8dd2644271d0138839490342833e9ff7f82772038f673f5ac6220193c587747d - name: git - version: 2.2.1 - sha: 4ba2185688a8e1ffdce43916448ff3a25a8ef845feebb3c95f47a0bb65e11252 + version: 2.2.5 + sha: 92c51f33fbcbe858d05b40083d3c628f03b6ba5218626ee22db9a367947b7670 - name: git-client - version: 1.8.0 - sha: 091df903bf1ed2b0c531714199ff8bb9225deaa2096520753554a39f2557d9e8 + version: 1.10.1 + sha: 19de6979a1360bc022bba9e061c4f946e51f252912234453d7f70af62d089e65 - name: instant-messaging - version: 1.28 - sha: 0b84561fd72cb80d89c5c57548fe8b7270d448f66361dedd07e227fb1bd44f03 + version: 1.29 + sha: b8fc1bff0c6f899f60d2d02b4ed321baf045fc0e5d4e0c3676d99197f94a8e5c - name: ldap version: 1.8 sha: 491905ec3675b6a5acf2098722c121732801fd6210e6ff54bc99d213b5b8ee58 - name: maven-plugin - version: 2.2 - sha: b373d99ffbdec45375fcf00be329d7b5029ab195f5b48d2d7518c776ed4bf1b8 + version: 2.6 + sha: 3a3a1e1d7e3416ea85ec09f953f5b8e37d943ca55b8e4224bbcfd702bed72fa5 - name: mercurial version: '1.50' sha: 934a6bd38e2109b97c915d80fdb6abc74a8ef4aff882b94ef0b1a274919ea407 - name: openid - version: 1.8 - sha: fed09c7da7762323cf55c3b725493622a4a2460eab8622230497e35914ac9d7e + version: 2.1 + sha: 7c0d2cd72fd2554a60a01fe6dc169c994487d6b228771b42d42c49559d388c4a - name: python version: 1.2 sha: e3358a945f21b84a8156237b0d621815a7822322e1180ae1e66d10798aaf1f56 @@ -154,17 +154,23 @@ version: 0.7.11 sha: f8eacb53eb01f83f3702009a41cef89e520a72933671ac1ee9154d88bde2d67a - name: xunit - version: 1.84 - sha: d06679ec0f3e2540615109789219404d602c98beda7be555dda7732a463c096b + version: 1.90 + sha: 2beade6d7769db9d52ff147c7a491cd1e7c53b01c07b9eeb44daa27ee75b25ca - name: multiple-scms version: 0.3 sha: e79d7e855ffe0ad060d11ae1ce0b39f68e7fa031c6e831f60fe33e5ddb3392ac - name: credentials - version: 1.9.4 - sha: 2fedc41d977a166c1addd82cd0cc9b73cffd34b97f7c0756bad7dc198ccd98de + version: 1.16.1 + sha: ae7e8ab317c03355390135d5eec683db7dceb5d513717d9fab624238a5ffe2bf - name: mailer - version: 1.8 - sha: fb9c6d471c2fea97fc2ccb64bfac18f77c847e740bcc2d5a4de31c35e851728a + version: 1.11 + sha: 9217be3008f323ac0535d4fb34118ed2681d6170d2d7de2f38b99ba331c4a256 + - name: matrix-auth + version: 1.2 + sha: a773c2fd6b2d70b2ff1c0466308290326d97f05b6fa72a217922997750aef39a + - name: javadoc + version: 1.2 + sha: 4bde54b288b24d5deaa7f809df78373d3b37d683d4693ab42278f019252c86b9 notify: - restart jenkins tags: From d0389d3de8d02ff605b10bf3d8220a8366b7ec5f Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 20 Aug 2014 07:47:28 +0000 Subject: [PATCH 083/193] buildmaster: trying to fix mail reporting --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index dbddad051e..d4bdb4f586 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -273,11 +273,11 @@ def html_message_formatter(mode, name, build, results, master_status): if log.getName() == 'stdio': break - name = "%s.%s" % (step.getName(), log.getName()) + name = "%s.%s" % (log.getStep().getName(), log.getName()) status, dummy = log.getStep().getResults() content = log.getText().splitlines() # Note: can be VERY LARGE url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), - step.getName(), log.getName()) + log.getStep().getName(), log.getName()) text.append(u'

') text.append(u'Detailed log of the %s build step: %s' From b30f78e2ad45a0d42e483f7d522913d72170adaa Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:10:33 +0200 Subject: [PATCH 084/193] 1.90 is 1.90, not 1.9 ... --- playbooks/groups/jenkins-cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 7943ef0309..f85595b157 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -154,7 +154,7 @@ version: 0.7.11 sha: f8eacb53eb01f83f3702009a41cef89e520a72933671ac1ee9154d88bde2d67a - name: xunit - version: 1.90 + version: '1.90' sha: 2beade6d7769db9d52ff147c7a491cd1e7c53b01c07b9eeb44daa27ee75b25ca - name: multiple-scms version: 0.3 From a7396c980a137a93a42d218d2a88955530c97c33 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:14:45 +0200 Subject: [PATCH 085/193] Let's just use quotes everywhere --- playbooks/groups/jenkins-cloud.yml | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index f85595b157..78e18e7e62 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -100,76 +100,76 @@ sha256sum={{ item.sha }} with_items: - name: bazaar - version: 1.22 + version: '1.22' sha: d7ff0987c96e2a694257ecf897ceee376908c5f94abfd1d5efc32482e4d54141 - name: chucknorris - version: 0.5 + version: '0.5' sha: bd9df0507008255ad2ed046368d10a4d039a6cbcfefb53c71c1768cc0dcbf65b - name: cobertura - version: 1.9.5 + version: '1.9.5' sha: a76bc1524efc5ba05672638001c0e951edd2a853d222efcfb035e02169e4252a - name: cvs - version: 2.12 + version: '2.12' sha: 6e6dfd35e8501bf5f84a9d43d210db61165ce51a606327fc81f2efc5208478ba - name: external-monitor-job - version: 1.2 + version: '1.2' sha: 8dd2644271d0138839490342833e9ff7f82772038f673f5ac6220193c587747d - name: git - version: 2.2.5 + version: '2.2.5' sha: 92c51f33fbcbe858d05b40083d3c628f03b6ba5218626ee22db9a367947b7670 - name: git-client - version: 1.10.1 + version: '1.10.1' sha: 19de6979a1360bc022bba9e061c4f946e51f252912234453d7f70af62d089e65 - name: instant-messaging - version: 1.29 + version: '1.29' sha: b8fc1bff0c6f899f60d2d02b4ed321baf045fc0e5d4e0c3676d99197f94a8e5c - name: ldap - version: 1.8 + version: '1.8' sha: 491905ec3675b6a5acf2098722c121732801fd6210e6ff54bc99d213b5b8ee58 - name: maven-plugin - version: 2.6 + version: '2.6' sha: 3a3a1e1d7e3416ea85ec09f953f5b8e37d943ca55b8e4224bbcfd702bed72fa5 - name: mercurial version: '1.50' sha: 934a6bd38e2109b97c915d80fdb6abc74a8ef4aff882b94ef0b1a274919ea407 - name: openid - version: 2.1 + version: '2.1' sha: 7c0d2cd72fd2554a60a01fe6dc169c994487d6b228771b42d42c49559d388c4a - name: python - version: 1.2 + version: '1.2' sha: e3358a945f21b84a8156237b0d621815a7822322e1180ae1e66d10798aaf1f56 - name: scm-api - version: 0.2 + version: '0.2' sha: cc856d8dc8b951cf9a195baa2bf7bbff0d12368534a6b973e43e2909141eff3f - name: ssh-agent - version: 1.4.1 + version: '1.4.1' sha: ae8227bf219e96a4d76f36dc6d6e652ddd0209e8d9c4cf4483a07858d707ce6e - name: subversion - version: 2.2 + version: '2.2' sha: 221ed61c8e4ef959bb316ea93d188e19c8f980edac0f1e45a6cd8d7e13808b51 - name: translation - version: 1.11 + version: '1.11' sha: 4d88b8d74ade119cef76827bd385693447fa68fa18fd1bfc8806aff9d931f00e - name: violations - version: 0.7.11 + version: '0.7.11' sha: f8eacb53eb01f83f3702009a41cef89e520a72933671ac1ee9154d88bde2d67a - name: xunit version: '1.90' sha: 2beade6d7769db9d52ff147c7a491cd1e7c53b01c07b9eeb44daa27ee75b25ca - name: multiple-scms - version: 0.3 + version: '0.3' sha: e79d7e855ffe0ad060d11ae1ce0b39f68e7fa031c6e831f60fe33e5ddb3392ac - name: credentials - version: 1.16.1 + version: '1.16.1' sha: ae7e8ab317c03355390135d5eec683db7dceb5d513717d9fab624238a5ffe2bf - name: mailer - version: 1.11 + version: '1.11' sha: 9217be3008f323ac0535d4fb34118ed2681d6170d2d7de2f38b99ba331c4a256 - name: matrix-auth - version: 1.2 + version: '1.2' sha: a773c2fd6b2d70b2ff1c0466308290326d97f05b6fa72a217922997750aef39a - name: javadoc - version: 1.2 + version: '1.2' sha: 4bde54b288b24d5deaa7f809df78373d3b37d683d4693ab42278f019252c86b9 notify: - restart jenkins From c76b047769cb76d59db82296db7a96a47fe2a1ff Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:16:03 +0200 Subject: [PATCH 086/193] One more place needing quotes --- playbooks/groups/jenkins-cloud.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 78e18e7e62..fe1683048c 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -182,7 +182,7 @@ sha256sum={{ item.sha }} with_items: - name: warnings - version: 4.39 + version: '4.39' sha: 7652b7ed8971de932f46323aa8e0ddee2bcf4f14839296481ae79590e09f7606 notify: - restart jenkins From 56d2bec5d9cb979204b5cca5bc97faffd8039004 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:26:01 +0200 Subject: [PATCH 087/193] Revert to the previous version of the working openid plugin --- playbooks/groups/jenkins-cloud.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index fe1683048c..539a2ca44d 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -133,8 +133,8 @@ version: '1.50' sha: 934a6bd38e2109b97c915d80fdb6abc74a8ef4aff882b94ef0b1a274919ea407 - name: openid - version: '2.1' - sha: 7c0d2cd72fd2554a60a01fe6dc169c994487d6b228771b42d42c49559d388c4a + version: '1.8' + sha: fed09c7da7762323cf55c3b725493622a4a2460eab8622230497e35914ac9d7e - name: python version: '1.2' sha: e3358a945f21b84a8156237b0d621815a7822322e1180ae1e66d10798aaf1f56 From 28765657544d95aa748c1bff7b807a28628ee881 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:43:49 +0200 Subject: [PATCH 088/193] Small formatting changes --- playbooks/groups/jenkins-cloud.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 539a2ca44d..76e775f089 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -54,7 +54,9 @@ - packages - name: add jenkins proxy config file for apache - action: copy src="{{ files }}/jenkins/master/jenkins-apache.conf" dest=/etc/httpd/conf.d/jenkins-apache.conf owner=root group=root mode=0644 + action: copy src="{{ files }}/jenkins/master/jenkins-apache.conf" + dest=/etc/httpd/conf.d/jenkins-apache.conf + owner=root group=root mode=0644 notify: - restart httpd tags: @@ -64,12 +66,16 @@ action: service name=httpd state=running enabled=true - name: add jenkins upstream repo - action: copy src="{{ files }}/jenkins/master/jenkins.repo" dest=/etc/yum.repos.d/jenkins.repo owner=root group=root + action: copy src="{{ files }}/jenkins/master/jenkins.repo" + dest=/etc/yum.repos.d/jenkins.repo + owner=root group=root tags: - config - name: import jenkins upstream gpg key - action: copy src="{{ files }}/jenkins/master/jenkins-ci.org.key" dest=/etc/pki/rpm-gpg/RPM-GPG-KEY-jenkins-ci.org owner=root group=root + action: copy src="{{ files }}/jenkins/master/jenkins-ci.org.key" + dest=/etc/pki/rpm-gpg/RPM-GPG-KEY-jenkins-ci.org + owner=root group=root tags: - config From 6b3c84aa7c42a247bc0c35121b8853f847d280c6 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Wed, 20 Aug 2014 10:49:27 +0200 Subject: [PATCH 089/193] Chown to jenkins /var/lib/jenkins/ --- playbooks/groups/jenkins-cloud.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/playbooks/groups/jenkins-cloud.yml b/playbooks/groups/jenkins-cloud.yml index 76e775f089..1ff1bd6e83 100644 --- a/playbooks/groups/jenkins-cloud.yml +++ b/playbooks/groups/jenkins-cloud.yml @@ -202,6 +202,10 @@ tags: - config + - name: Give the user jenkins the ownership of the /var/lib/jenkins + file: path=/var/lib/jenkins/ + owner=jenkins group=jenkins recurse=yes + - name: add jenkins ssh priv key so it can connect to clients action: copy src="{{ private }}/files/jenkins/ssh/jenkins_master" dest=/var/tmp/jenkins_master_id_rsa mode=600 owner=jenkins group=jenkins tags: From 4f96d0905435b36fda937762e25c6ab52b5b56f3 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 20 Aug 2014 11:44:39 +0000 Subject: [PATCH 090/193] buildmaster: make step names in waterfall more readable --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index d4bdb4f586..8b392049b2 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -141,7 +141,7 @@ factory = BuildFactory() factory.addStep(RemoveDirectory(dir="/var/tmp/taskotron/")) # clean the log (see T230) -factory.addStep(ShellCommand(command=["rm", "-f", "/var/log/taskotron/taskotron.log"], name="rm_log")) +factory.addStep(ShellCommand(command=["rm", "-f", "/var/log/taskotron/taskotron.log"], name="rm_log", descriptionDone=['Clean log'])) # check out the source factory.addStep(Git(repourl=Interpolate('{{ grokmirror_user }}@{{ buildmaster }}:/var/lib/git/mirror/fedoraqa/%(prop:taskname)s/'), @@ -154,10 +154,12 @@ factory.addStep(ShellCommand(command=["runtask", '-i', '-a', Interpolate('%(prop:arch)s'), '-j', Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'), Interpolate('%(prop:taskname)s.yml')], + descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)')], name='runtask')) # capture the taskotron log -factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log")) +factory.addStep(ShellCommand(command=["cat", "/var/log/taskotron/taskotron.log"], name="cat_log", + descriptionDone=['Store log'])) ####### RESOURCE LOCKS From 1780914f5bc75b2e992bd2ade1230ba033b49540 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Wed, 20 Aug 2014 11:50:31 +0000 Subject: [PATCH 091/193] Fix typo... --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 8b392049b2..8e8f8c8f6e 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -154,7 +154,7 @@ factory.addStep(ShellCommand(command=["runtask", '-i', '-a', Interpolate('%(prop:arch)s'), '-j', Interpolate('%(prop:buildername)s/%(prop:buildnumber)s'), Interpolate('%(prop:taskname)s.yml')], - descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)')], + descriptionDone=[Interpolate('%(prop:taskname)s on %(prop:item)s')], name='runtask')) # capture the taskotron log From ce9a4403241e27e189e62198fbd9be3e7aad9bee Mon Sep 17 00:00:00 2001 From: Tim Flink Date: Wed, 20 Aug 2014 17:42:28 +0000 Subject: [PATCH 092/193] commenting out db init in resultsdb-be for now since it's always being run and clears contents of the db --- roles/taskotron/resultsdb-backend/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/resultsdb-backend/tasks/main.yml b/roles/taskotron/resultsdb-backend/tasks/main.yml index 1aa4a81401..c3833476f2 100644 --- a/roles/taskotron/resultsdb-backend/tasks/main.yml +++ b/roles/taskotron/resultsdb-backend/tasks/main.yml @@ -47,5 +47,8 @@ notify: - restart httpd -- name: initialize resultsdb database - shell: PROD='true' resultsdb init_db && touch /etc/resultsdb/db-is-init creates=/etc/resultsdb-is-init +# this command will clear the exising database and it's now being run every time, regardless of whether +# /etc/resultsdb-is-init exists or not. We've already lost a lot of data, so commenting this command out +# while we figure out how to handle the problem +#- name: initialize resultsdb database +# shell: PROD='true' resultsdb init_db && touch /etc/resultsdb/db-is-init creates=/etc/resultsdb-is-init From b5425e166b7a249ed5bd1bf37c48763b9542da5a Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Wed, 20 Aug 2014 18:36:46 +0000 Subject: [PATCH 093/193] we need the docker plugin --- roles/koji_builder/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/koji_builder/tasks/main.yml b/roles/koji_builder/tasks/main.yml index 563be3db01..7d66d847b1 100644 --- a/roles/koji_builder/tasks/main.yml +++ b/roles/koji_builder/tasks/main.yml @@ -64,6 +64,7 @@ - oz - imagefactory - imagefactory-plugins-TinMan + - imagefactory-plugins-Docker - imagefactory-plugins-vSphere - imagefactory-plugins-ovfcommon - imagefactory-plugins From 44f2830e17c7bc256e4738a5c08652e4828836da Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 21 Aug 2014 08:47:06 +0000 Subject: [PATCH 094/193] buildmaster: include taskname in the message body --- .../templates/taskotron.master.cfg.j2 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 8e8f8c8f6e..71ef9efb6f 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -278,6 +278,11 @@ def html_message_formatter(mode, name, build, results, master_status): name = "%s.%s" % (log.getStep().getName(), log.getName()) status, dummy = log.getStep().getResults() content = log.getText().splitlines() # Note: can be VERY LARGE + + if step.getName() == 'runtask': + taskname = re.search("task=\['(.*)\.yml'\]", content).group(1) + text.insert(0, '

Task name: %s

' % taskname) + url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), log.getStep().getName(), log.getName()) @@ -297,8 +302,6 @@ def html_message_formatter(mode, name, build, results, master_status): text.append(u'

') text.append(u'-The Buildbot') - taskname = re.search("task=\['(.*)\.yml'\]", t).group(1) - test.insert(0, '

Task name: %s

' % taskname) return { 'body': u"\n".join(text), From 7a8b0690406652ad3a8c8387b6e6da96ab5b9036 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Thu, 21 Aug 2014 13:17:45 +0000 Subject: [PATCH 095/193] Update playbook list --- master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master.yml b/master.yml index cb2114343a..d66bd5d8f1 100644 --- a/master.yml +++ b/master.yml @@ -15,7 +15,6 @@ - include: /srv/web/infra/ansible/playbooks/groups/arm-packager.yml - include: /srv/web/infra/ansible/playbooks/groups/arm-qa.yml - include: /srv/web/infra/ansible/playbooks/groups/arm-releng.yml -- include: /srv/web/infra/ansible/playbooks/groups/arm-retrace.yml - include: /srv/web/infra/ansible/playbooks/groups/ask.yml - include: /srv/web/infra/ansible/playbooks/groups/atomic.yml - include: /srv/web/infra/ansible/playbooks/groups/autosign.yml @@ -58,6 +57,7 @@ - include: /srv/web/infra/ansible/playbooks/groups/resultsdb-prod.yml - include: /srv/web/infra/ansible/playbooks/groups/resultsdb-dev.yml - include: /srv/web/infra/ansible/playbooks/groups/resultsdb-stg.yml +- include: /srv/web/infra/ansible/playbooks/groups/retrace.yml - include: /srv/web/infra/ansible/playbooks/groups/smtp-mm.yml - include: /srv/web/infra/ansible/playbooks/groups/summershum.yml - include: /srv/web/infra/ansible/playbooks/groups/sundries.yml From aa06685f03f3f553fef77268c544ff596d6801c4 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Thu, 21 Aug 2014 15:47:03 +0000 Subject: [PATCH 096/193] Disable the cron.daily/epylog.cron as well --- roles/epylog/tasks/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/epylog/tasks/main.yml b/roles/epylog/tasks/main.yml index 46337a84c2..6c07012bd9 100644 --- a/roles/epylog/tasks/main.yml +++ b/roles/epylog/tasks/main.yml @@ -23,7 +23,10 @@ - modules/kojiload_mod.py - name: epylog cron - copy: src=epylog-default.cron dest=/etc/cron.d/epylog.cron + copy: src=epylog-default.cron dest=/etc/{{ item }}/epylog.cron + with_items: + - cron.d + - cron.daily tags: - config - cron From 613b791b1d35adc40d9f13a2d7395eadd84cbdf4 Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 21 Aug 2014 15:55:04 +0000 Subject: [PATCH 097/193] taskotron-trigger: schedule tasks only on single arch for now --- roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2 b/roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2 index c3ecc8e10a..f8b76032e3 100644 --- a/roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2 +++ b/roles/taskotron/taskotron-trigger/templates/trigger.cfg.j2 @@ -6,10 +6,10 @@ url = http://127.0.0.1:8080/change_hook [trigger] koji_build_completed_tasks = rpmlint koji_tag_changed_tasks = upgradepath,depcheck -valid_arches = i386,i686,x86_64,noarch +valid_arches = x86_64 koji_url = http://koji.fedoraproject.org/kojihub type = BuildbotTrigger -blacklist_releases = el5,el6 +blacklist_releases = el5,el6,el7 job_logging = True joblog_file = {{ trigger_joblog_file }} fuse_delay = 900 ; 15 minutes in seconds From 642ea7df2c7b39be8a11fa10d065ebf90dacfb1b Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Thu, 21 Aug 2014 16:41:25 +0000 Subject: [PATCH 098/193] lets use a better httpd config --- roles/releng/files/compose.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/releng/files/compose.conf b/roles/releng/files/compose.conf index a174ab306c..9b746e79da 100644 --- a/roles/releng/files/compose.conf +++ b/roles/releng/files/compose.conf @@ -1,8 +1,7 @@ -Alias /compose /srv/pungi/ +Alias /compose /mnt/fedora_koji/compose/ - - Order deny,allow - Allow from all + + Options +FollowSymLinks +Indexes + Require all granted - From 3f08b61023751bc3270f95f3635d7df1479346d9 Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Thu, 21 Aug 2014 17:06:40 +0000 Subject: [PATCH 099/193] allow outbound http and https to the compose box from the builders --- roles/base/templates/iptables/iptables.kojibuilder | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/base/templates/iptables/iptables.kojibuilder b/roles/base/templates/iptables/iptables.kojibuilder index 439db80b53..858b10c657 100644 --- a/roles/base/templates/iptables/iptables.kojibuilder +++ b/roles/base/templates/iptables/iptables.kojibuilder @@ -31,6 +31,10 @@ -A OUTPUT -p tcp -m tcp -d 10.5.124.138 --dport 80 -j ACCEPT -A OUTPUT -p tcp -m tcp -d 10.5.124.138 --dport 443 -j ACCEPT +# compose-x86-02.fp.o +-A OUTPUT -p tcp -m tcp -d 10.5.125.42 --dport 80 -j ACCEPT +-A OUTPUT -p tcp -m tcp -d 10.5.125.42 --dport 443 -j ACCEPT + # DNS -A OUTPUT -p udp -m udp -d 10.5.126.21 --dport 53 -j ACCEPT -A OUTPUT -p udp -m udp -d 10.5.126.22 --dport 53 -j ACCEPT From b2a26bf3bfe8a62798cd5483da3fc2ef90cd0347 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Aug 2014 17:47:10 +0000 Subject: [PATCH 100/193] Add some of the newer fedmsg categories to nagios. --- .../templates/check_datanommer_history.cfg.j2 | 8 +++- .../files/nagios/services/fedmsg.cfg | 43 ++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 index fb563c5413..5bd99c9500 100644 --- a/roles/nagios_client/templates/check_datanommer_history.cfg.j2 +++ b/roles/nagios_client/templates/check_datanommer_history.cfg.j2 @@ -30,8 +30,14 @@ command[check_datanommer_fedbadges]={{libdir}}/nagios/plugins/check_datanommer_t command[check_datanommer_nuancier]={{libdir}}/nagios/plugins/check_datanommer_timesince.py nuancier 23652000 31536000 command[check_datanommer_fedocal]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedocal 7884000 23652000 command[check_datanommer_ansible]={{libdir}}/nagios/plugins/check_datanommer_timesince.py ansible 432000 604800 +command[check_datanommer_summershum]={{libdir}}/nagios/plugins/check_datanommer_timesince.py summershum 604800 1814400 +command[check_datanommer_jenkins]={{libdir}}/nagios/plugins/check_datanommer_timesince.py jenkins 432000 604800 +command[check_datanommer_github]={{libdir}}/nagios/plugins/check_datanommer_timesince.py github 432000 604800 +command[check_datanommer_kerneltest]={{libdir}}/nagios/plugins/check_datanommer_timesince.py kerneltest 604800 1814400 +command[check_datanommer_fmn]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fmn 604800 1814400 # These are not actually finished and deployed yet command[check_datanommer_mailman]={{libdir}}/nagios/plugins/check_datanommer_timesince.py mailman 14400 86400 command[check_datanommer_cnucnuweb]={{libdir}}/nagios/plugins/check_datanommer_timesince.py cnucnuweb 604800 1814400 -command[check_datanommer_summershum]={{libdir}}/nagios/plugins/check_datanommer_timesince.py summershum 604800 1814400 +command[check_datanommer_fedimg]={{libdir}}/nagios/plugins/check_datanommer_timesince.py fedimg 259200 604800 +command[check_datanommer_bugzilla]={{libdir}}/nagios/plugins/check_datanommer_timesince.py bugzilla 86400 259200 diff --git a/roles/nagios_server/files/nagios/services/fedmsg.cfg b/roles/nagios_server/files/nagios/services/fedmsg.cfg index c4fbe81931..486216cc99 100644 --- a/roles/nagios_server/files/nagios/services/fedmsg.cfg +++ b/roles/nagios_server/files/nagios/services/fedmsg.cfg @@ -178,10 +178,49 @@ define service { #} #define service { # host_name busgateway01 -# service_description Check datanommer for recent summershum messages -# check_command check_by_nrpe!check_datanommer_summershum +# service_description Check datanommer for recent fedimg messages +# check_command check_by_nrpe!check_datanommer_fedimg # use defaulttemplate #} +#define service { +# host_name busgateway01 +# service_description Check datanommer for recent bugzilla messages +# check_command check_by_nrpe!check_datanommer_bugzilla +# use defaulttemplate +#} +define service { + host_name busgateway01 + service_description Check datanommer for recent summershum messages + check_command check_by_nrpe!check_datanommer_summershum + use defaulttemplate +} +define service { + host_name busgateway01 + service_description Check datanommer for recent jenkins messages + check_command check_by_nrpe!check_datanommer_jenkins + use defaulttemplate +} +define service { + host_name busgateway01 + service_description Check datanommer for recent github messages + check_command check_by_nrpe!check_datanommer_github + use defaulttemplate +} +define service { + host_name busgateway01 + service_description Check datanommer for recent kerneltest messages + check_command check_by_nrpe!check_datanommer_kerneltest + use defaulttemplate +} +define service { + host_name busgateway01 + service_description Check datanommer for recent fmn messages + check_command check_by_nrpe!check_datanommer_fmn + use defaulttemplate +} + + + define service { host_name busgateway01 From 78c78b9f6b43eac0ec1419743cfa118dfd683334 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Thu, 21 Aug 2014 17:51:37 +0000 Subject: [PATCH 101/193] Add nagios_server tag to the nagios_server role. --- roles/nagios_server/tasks/main.yml | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/roles/nagios_server/tasks/main.yml b/roles/nagios_server/tasks/main.yml index fba4f76565..3f0fe85ce6 100644 --- a/roles/nagios_server/tasks/main.yml +++ b/roles/nagios_server/tasks/main.yml @@ -1,13 +1,19 @@ - name: Add nagios group as 420 group: name=nagios gid=420 state=present system=yes + tags: + - nagios_server - name: Add nagios user as 420 user: name=nagios uid=420 state=present home=/var/spool/nagios comment="Nagios Monitoring User" createhome=yes system=yes shell=/bin/bash + tags: + - nagios_server # Add the apache user to the nagios group so that it has permissions # to do stuff in /var/spool/nagios/* - name: Add apache to nagios group user: name=apache append=yes groups=nagios + tags: + - nagios_server - name: Install nagios packages yum: name={{ item }} state=installed @@ -28,35 +34,54 @@ - nagios-plugins-dummy - stunnel - mod_auth_openid + tags: + - nagios_server - name: Copy /etc/nagios config copy: src={{nagios_srcdir}}/ dest=/etc/nagios owner=nagios group=nagios tags: - nagios-config + - nagios_server - name: Copy checkcommands.cfg template: src={{nagios_srcdir}}/checkcommands.cfg dest=/etc/nagios/checkcommands.cfg owner=nagios group=nagios + tags: + - nagios_server - name: Copy httpd config template: src=nagios-httpd.conf dest=/etc/httpd/conf.d/nagios.conf + tags: + - nagios_server - name: Create eventhandlers directory file: dest=/usr/lib64/nagios/plugins/eventhandlers/ state=directory + tags: + - nagios_server - name: Copy plugins copy: src=plugins/ dest=/usr/lib64/nagios/plugins/ mode=0755 owner=root group=root + tags: + - nagios_server - name: Create log directory file: dest=/var/log/nagios state=directory group=nagios owner=nagios mode=0755 + tags: + - nagios_server - name: Create spool directory file: dest=/var/log/nagios/spool state=directory group=nagios owner=nagios mode=0755 + tags: + - nagios_server - name: Create checkresults directory file: dest=/var/log/nagios/spool/checkresults state=directory group=nagios owner=nagios mode=0755 + tags: + - nagios_server - name: Copy irc-colorize.py copy: src=irc-colorize.py dest=/usr/local/bin/irc-colorize.py mode=0755 group=root owner=root + tags: + - nagios_server - name: Copy specialized nrpe.cfg for nagios server copy: src=nrpe.cfg dest=/etc/nagios/nrpe.cfg mode=0644 group=root owner=root @@ -68,31 +93,47 @@ - name: Copy check_nagios_notifications.py copy: src=check_nagios_notifications.py dest=/usr/local/bin/check_nagios_notifications.py mode=0755 group=root owner=root + tags: + - nagios_server - name: Nuke default nagios passwd file: dest=/etc/nagios/passwd state=absent + tags: + - nagios_server - name: Nuke default nagios objects file: dest=/etc/nagios/objects state=absent + tags: + - nagios_server - name: Install check_nagios_notifications cron cron: name="check_nagios_notifications" minute=0 hour=0 weekday=1 user=nagios job="/usr/local/bin/check_nagios_notifications.py" + tags: + - nagios_server - name: Override config.inc.php for the given environment template: src=config.inc.php dest=/usr/share/nagios/html/config.inc.php mode=0640 owner=root group=apache + tags: + - nagios_server # Handle selinux annoyances - roughly copied from fedmsg role - name: Ensure a directory exists for our custom selinux module file: dest=/usr/local/share/nagios-policy state=directory + tags: + - nagios_server - name: Copy over our custom selinux module copy: src=selinux/hostname.pp dest=/usr/local/share/nagios-policy/hostname.pp register: selinux_module + tags: + - nagios_server - name: Install our custom selinux module command: semodule -i /usr/local/share/nagios-policy/hostname.pp when: selinux_module|changed + tags: + - nagios_server - name: Start and autostart services service: name={{item}} state=started enabled=yes @@ -102,3 +143,5 @@ - nrpe - postfix - rsyslog + tags: + - nagios_server From 40b1d0a809a5a28ac7aad9501d175518a3f182ac Mon Sep 17 00:00:00 2001 From: Dennis Gilmore Date: Thu, 21 Aug 2014 19:09:26 +0000 Subject: [PATCH 102/193] put the arm-releng boxes, i.e. compose boxes into the releng host group --- inventory/group_vars/arm-releng | 1 + 1 file changed, 1 insertion(+) diff --git a/inventory/group_vars/arm-releng b/inventory/group_vars/arm-releng index 758e7d70d3..f303db12e8 100644 --- a/inventory/group_vars/arm-releng +++ b/inventory/group_vars/arm-releng @@ -1,4 +1,5 @@ --- +host_group: releng fas_client_groups: sysadmin-releng freezes: false # From dce2f3f240caa830ad0cce7b24d3af19a83c980f Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Fri, 22 Aug 2014 07:37:41 +0000 Subject: [PATCH 103/193] buildmaster: search for a taskname only on first line --- .../buildmaster-configure/templates/taskotron.master.cfg.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 index 71ef9efb6f..e8d0d66105 100644 --- a/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 +++ b/roles/taskotron/buildmaster-configure/templates/taskotron.master.cfg.j2 @@ -280,7 +280,7 @@ def html_message_formatter(mode, name, build, results, master_status): content = log.getText().splitlines() # Note: can be VERY LARGE if step.getName() == 'runtask': - taskname = re.search("task=\['(.*)\.yml'\]", content).group(1) + taskname = re.search("task=\['(.*)\.yml'\]", content[0]).group(1) text.insert(0, '

Task name: %s

' % taskname) url = u'%s/steps/%s/logs/%s' % (master_status.getURLForThing(build), From 6fa000be7f271722e732396c46efe2bde4701f67 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 22 Aug 2014 16:16:34 +0200 Subject: [PATCH 104/193] Add playbook to run pkgdb2branch on pkgs01 --- playbooks/run_pkgdb2branch.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 playbooks/run_pkgdb2branch.yml diff --git a/playbooks/run_pkgdb2branch.yml b/playbooks/run_pkgdb2branch.yml new file mode 100644 index 0000000000..52e56ec4d8 --- /dev/null +++ b/playbooks/run_pkgdb2branch.yml @@ -0,0 +1,16 @@ +# Run `pkgdb2branch` on +# +#To update from testing, adjust as follow: +# --extra-vars="package=" + + +- name: run pkgdb2branch for the specified packages + hosts: pkgs01.phx2.fedoraproject.org + user: root + serial: 25 + gather_facts: False + + tasks: + - name: call pkgdb2branch.py on {{ package }} + command: /usr/local/bin/pkgdb2branch.py {{ package }} + From 4a37364d85aa422b05b99b8e6c498f6e9141d485 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Fri, 22 Aug 2014 14:25:44 +0000 Subject: [PATCH 105/193] Add db-datanommer01 to /etc/hosts in staging. --- roles/hosts/files/staging-hosts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hosts/files/staging-hosts b/roles/hosts/files/staging-hosts index ec41150778..d70289c8f9 100644 --- a/roles/hosts/files/staging-hosts +++ b/roles/hosts/files/staging-hosts @@ -5,6 +5,6 @@ 10.5.126.88 proxy01.phx2.fedoraproject.org proxy1 proxy2 proxy3 proxy4 proxy01 proxy02 proxy03 proxy04 fedoraproject.org 10.5.126.23 infrastructure.fedoraproject.org 10.5.126.81 app01.stg.fedoraproject.org memcached01 memcached02 memcached03 memcached04 -10.5.126.85 db02.stg.fedoraproject.org db01 db04 db05 db-ask db-datanommer db-tahrir db-elections db-fedocal db-github2fedmsg db-kerneltest db-notifs db-datanommer nuancier_db db-pkgdb2 db-summershum tagger_db +10.5.126.85 db02.stg.fedoraproject.org db01 db04 db05 db-ask db-datanommer db-datanommer01 db-tahrir db-elections db-fedocal db-github2fedmsg db-kerneltest db-notifs nuancier_db db-pkgdb2 db-summershum tagger_db 10.5.126.86 fas01.phx2.fedoraproject.org fas1 fas2 fas01 fas02 fas03 fas-all 10.5.126.87 koji01.stg.phx2.fedoraproject.org koji.stg.fedoraproject.org koji01 kojipkgs kojipkgs.stg.phx2.fedoraproject.org kojipkgs.stg.fedoraproject.org From 771b5f38bd922d01894918aa130f9e6a0b9e1be4 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Fri, 22 Aug 2014 16:31:15 +0200 Subject: [PATCH 106/193] Fix documentation and invoking pkgdb2branch for multiple packages --- playbooks/run_pkgdb2branch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/run_pkgdb2branch.yml b/playbooks/run_pkgdb2branch.yml index 52e56ec4d8..bdb20e01fd 100644 --- a/playbooks/run_pkgdb2branch.yml +++ b/playbooks/run_pkgdb2branch.yml @@ -1,7 +1,7 @@ # Run `pkgdb2branch` on # #To update from testing, adjust as follow: -# --extra-vars="package=" +# --extra-vars="package='pkg1;pkg2;pkg3'" - name: run pkgdb2branch for the specified packages @@ -12,5 +12,5 @@ tasks: - name: call pkgdb2branch.py on {{ package }} - command: /usr/local/bin/pkgdb2branch.py {{ package }} + command: /usr/local/bin/pkgdb2branch.py {{ package | join(" ") }} From a642d427aff93ba2142ac83291414868bf6f057c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Mon, 25 Aug 2014 09:26:49 +0000 Subject: [PATCH 107/193] use https for copr --- files/copr/copr-be.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/copr/copr-be.conf b/files/copr/copr-be.conf index fa138451aa..7828cb9701 100644 --- a/files/copr/copr-be.conf +++ b/files/copr/copr-be.conf @@ -2,12 +2,12 @@ # URL where are results visible # default is http://copr -results_baseurl=http://copr-be.cloud.fedoraproject.org/results +results_baseurl=https://copr-be.cloud.fedoraproject.org/results # ??? What is this # default is http://coprs/rest/api #frontend_url=http://copr-fe.cloud.fedoraproject.org/backend -frontend_url=http://172.16.5.31/backend +frontend_url=https://172.16.5.31/backend # must have same value as BACKEND_PASSWORD from have frontend in /etc/copr/copr.conf # default is PASSWORDHERE but you really should change it. really. From 63649415c597e8c511bd29926304c1676b4ea4e1 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Mon, 25 Aug 2014 16:35:18 +0000 Subject: [PATCH 108/193] and then we made ppc-comm04 boot from net-bios --- .../files/dhcpd.conf.noc01.phx2.fedoraproject.org | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 0624ece83e..9bb2c2e7b2 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -1452,6 +1452,14 @@ shared-network qa { filename "yaboot"; } + host ppc-comm04.qa.fedoraproject.org{ + hardware ethernet 5c:f3:fc:89:bd:c0; + fixed-address 10.5.124.221; + next-server ppc-builder7.qa.fedoraproject.org; + option host-name "ppc-comm04"; + filename "ppccomm04"; + } + host qa01 { hardware ethernet 00:21:5E:C7:5C:84; From 43c5b47eb6d3705806e552a8db081c960c048819 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 00:14:22 +0000 Subject: [PATCH 109/193] lets add the initial inventory for our cloud noc box --- inventory/host_vars/cloud-noc01.phx2.fedoraproject.org | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 inventory/host_vars/cloud-noc01.phx2.fedoraproject.org diff --git a/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org new file mode 100644 index 0000000000..83e0f1c807 --- /dev/null +++ b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org @@ -0,0 +1,9 @@ +--- +nrpe_procs_warn: 900 +nrpe_procs_crit: 1000 +datacenter: cloud +nm: 255.255.255.0 +gw: 209.132.184.254 +dns: 8.8.8.8 +eth0_ip: 209.132.184.17 +eth1_ip: 172.23.0.17 From 1005be1d5cd45492a90c232727b2f05e4630fa85 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 00:31:17 +0000 Subject: [PATCH 110/193] make a minimal playbook for cloud-noc --- .../cloud-noc01.phx2.fedoraproject.org | 1 + .../cloud-noc01.cloud.fedoraproject.org.yml | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml diff --git a/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org index 83e0f1c807..dbccb997f9 100644 --- a/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org +++ b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org @@ -4,6 +4,7 @@ nrpe_procs_crit: 1000 datacenter: cloud nm: 255.255.255.0 gw: 209.132.184.254 +fas_client_groups: sysadmin-main dns: 8.8.8.8 eth0_ip: 209.132.184.17 eth1_ip: 172.23.0.17 diff --git a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml new file mode 100644 index 0000000000..4705766875 --- /dev/null +++ b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml @@ -0,0 +1,30 @@ +# This is a basic playbook + +- name: make cloud noc hardware + user: root + accelerate: "{{ accelerated }}" + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - { role: denyhosts, when: ansible_distribution_major_version != '7' } + - nagios_client + - hosts + - fas_client + - collectd/base + - sudo + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + From 8a984f0960b8b9ce2ba3efba3ffe669cd2c0e29f Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 00:33:18 +0000 Subject: [PATCH 111/193] oh look we forgot a hosts line. --- playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml index 4705766875..a9f1af51b6 100644 --- a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml @@ -1,6 +1,7 @@ # This is a basic playbook - name: make cloud noc hardware + hosts: cloud-noc01.fedoraproject.org user: root accelerate: "{{ accelerated }}" gather_facts: True From dc5a13019d4a8bd099dd64aca92d0ff55c04a947 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 00:38:05 +0000 Subject: [PATCH 112/193] add cloud hardware for inventory --- inventory/inventory | 19 +++++++++++++++++++ .../cloud-noc01.cloud.fedoraproject.org.yml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/inventory/inventory b/inventory/inventory index 5d97721832..8dbfc78ab4 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -577,6 +577,25 @@ wiki01.stg.phx2.fedoraproject.org wiki01.phx2.fedoraproject.org wiki02.phx2.fedoraproject.org +[cloud-hardware] +fed-cloud01.cloud.fedoraproject.org +fed-cloud02.cloud.fedoraproject.org +fed-cloud03.cloud.fedoraproject.org +fed-cloud04.cloud.fedoraproject.org +fed-cloud05.cloud.fedoraproject.org +fed-cloud06.cloud.fedoraproject.org +fed-cloud07.cloud.fedoraproject.org +fed-cloud08.cloud.fedoraproject.org +fed-cloud09.cloud.fedoraproject.org +fed-cloud10.cloud.fedoraproject.org +fed-cloud11.cloud.fedoraproject.org +fed-cloud12.cloud.fedoraproject.org +fed-cloud13.cloud.fedoraproject.org +fed-cloud14.cloud.fedoraproject.org +fed-cloud15.cloud.fedoraproject.org +fed-cloud16.cloud.fedoraproject.org +cloud-noc01.cloud.fedoraproject.org + [persistent-cloud] #fedocal.dev.fedoraproject.org 209.132.184.147 diff --git a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml index a9f1af51b6..eac7d5c7fd 100644 --- a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml @@ -1,7 +1,7 @@ # This is a basic playbook - name: make cloud noc hardware - hosts: cloud-noc01.fedoraproject.org + hosts: 209.132.184.17 user: root accelerate: "{{ accelerated }}" gather_facts: True From 2dd613882ff650f4de1c47d81cd9aebb6fbe12d0 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 00:39:23 +0000 Subject: [PATCH 113/193] wheee. forgot to use name and not ip --- playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml index eac7d5c7fd..8d16831f03 100644 --- a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml @@ -1,7 +1,7 @@ # This is a basic playbook - name: make cloud noc hardware - hosts: 209.132.184.17 + hosts: cloud-noc01.cloud.fedoraproject.org user: root accelerate: "{{ accelerated }}" gather_facts: True From b121d21d569d3cddf0a95183b24afb4be6eba690 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 13:53:15 +0000 Subject: [PATCH 114/193] These machines don't exist yet. --- inventory/inventory | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/inventory/inventory b/inventory/inventory index 8dbfc78ab4..767e188f2d 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -589,11 +589,11 @@ fed-cloud08.cloud.fedoraproject.org fed-cloud09.cloud.fedoraproject.org fed-cloud10.cloud.fedoraproject.org fed-cloud11.cloud.fedoraproject.org -fed-cloud12.cloud.fedoraproject.org -fed-cloud13.cloud.fedoraproject.org -fed-cloud14.cloud.fedoraproject.org -fed-cloud15.cloud.fedoraproject.org -fed-cloud16.cloud.fedoraproject.org +#fed-cloud12.cloud.fedoraproject.org +#fed-cloud13.cloud.fedoraproject.org +#fed-cloud14.cloud.fedoraproject.org +#fed-cloud15.cloud.fedoraproject.org +#fed-cloud16.cloud.fedoraproject.org cloud-noc01.cloud.fedoraproject.org [persistent-cloud] From fed72f7ba11ee89520f289498bd85aa23b87e69a Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Fri, 4 Jul 2014 17:59:44 +0200 Subject: [PATCH 115/193] Add a new git/hooks role This will be needed to migrate Dist Git from puppet to ansible. --- roles/git/hooks/files/git.py | 211 ++++ .../git/hooks/files/gnome-post-receive-email | 941 ++++++++++++++++++ roles/git/hooks/files/post-received-chained | 8 + roles/git/hooks/files/post-received-fedmsg | 65 ++ roles/git/hooks/files/util.py | 153 +++ roles/git/hooks/tasks/main.yml | 22 + 6 files changed, 1400 insertions(+) create mode 100644 roles/git/hooks/files/git.py create mode 100644 roles/git/hooks/files/gnome-post-receive-email create mode 100644 roles/git/hooks/files/post-received-chained create mode 100644 roles/git/hooks/files/post-received-fedmsg create mode 100644 roles/git/hooks/files/util.py create mode 100644 roles/git/hooks/tasks/main.yml diff --git a/roles/git/hooks/files/git.py b/roles/git/hooks/files/git.py new file mode 100644 index 0000000000..72adff1f7c --- /dev/null +++ b/roles/git/hooks/files/git.py @@ -0,0 +1,211 @@ +# Utility functions for git +# +# Copyright (C) 2008 Owen Taylor +# Copyright (C) 2009 Red Hat, Inc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, If not, see +# http://www.gnu.org/licenses/. +# +# (These are adapted from git-bz) + +import os +import re +from subprocess import Popen, PIPE +import sys + +from util import die + +# Clone of subprocess.CalledProcessError (not in Python 2.4) +class CalledProcessError(Exception): + def __init__(self, returncode, cmd): + self.returncode = returncode + self.cmd = cmd + + def __str__(self): + return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode) + +NULL_REVISION = "0000000000000000000000000000000000000000" + +# Run a git command +# Non-keyword arguments are passed verbatim as command line arguments +# Keyword arguments are turned into command line options +# =True => -- +# ='' => --= +# Special keyword arguments: +# _quiet: Discard all output even if an error occurs +# _interactive: Don't capture stdout and stderr +# _input=: Feed to stdinin of the command +# _outfile= as the output file descriptor +# _split_lines: Return an array with one string per returned line +# +def git_run(command, *args, **kwargs): + to_run = ['git', command.replace("_", "-")] + + interactive = False + quiet = False + input = None + interactive = False + outfile = None + do_split_lines = False + for (k,v) in kwargs.iteritems(): + if k == '_quiet': + quiet = True + elif k == '_interactive': + interactive = True + elif k == '_input': + input = v + elif k == '_outfile': + outfile = v + elif k == '_split_lines': + do_split_lines = True + elif v is True: + if len(k) == 1: + to_run.append("-" + k) + else: + to_run.append("--" + k.replace("_", "-")) + else: + to_run.append("--" + k.replace("_", "-") + "=" + v) + + to_run.extend(args) + + if outfile: + stdout = outfile + else: + if interactive: + stdout = None + else: + stdout = PIPE + + if interactive: + stderr = None + else: + stderr = PIPE + + if input != None: + stdin = PIPE + else: + stdin = None + + process = Popen(to_run, + stdout=stdout, stderr=stderr, stdin=stdin) + output, error = process.communicate(input) + if process.returncode != 0: + if not quiet and not interactive: + print >>sys.stderr, error, + print output, + raise CalledProcessError(process.returncode, " ".join(to_run)) + + if interactive or outfile: + return None + else: + if do_split_lines: + return output.strip().splitlines() + else: + return output.strip() + +# Wrapper to allow us to do git.(...) instead of git_run() +class Git: + def __getattr__(self, command): + def f(*args, **kwargs): + return git_run(command, *args, **kwargs) + return f + +git = Git() + +class GitCommit: + def __init__(self, id, subject): + self.id = id + self.subject = subject + +# Takes argument like 'git.rev_list()' and returns a list of commit objects +def rev_list_commits(*args, **kwargs): + kwargs_copy = dict(kwargs) + kwargs_copy['pretty'] = 'format:%s' + kwargs_copy['_split_lines'] = True + lines = git.rev_list(*args, **kwargs_copy) + if (len(lines) % 2 != 0): + raise RuntimeException("git rev-list didn't return an even number of lines") + + result = [] + for i in xrange(0, len(lines), 2): + m = re.match("commit\s+([A-Fa-f0-9]+)", lines[i]) + if not m: + raise RuntimeException("Can't parse commit it '%s'", lines[i]) + commit_id = m.group(1) + subject = lines[i + 1] + result.append(GitCommit(commit_id, subject)) + + return result + +# Loads a single commit object by ID +def load_commit(commit_id): + return rev_list_commits(commit_id + "^!")[0] + +# Return True if the commit has multiple parents +def commit_is_merge(commit): + if isinstance(commit, basestring): + commit = load_commit(commit) + + parent_count = 0 + for line in git.cat_file("commit", commit.id, _split_lines=True): + if line == "": + break + if line.startswith("parent "): + parent_count += 1 + + return parent_count > 1 + +# Return a short one-line summary of the commit +def commit_oneline(commit): + if isinstance(commit, basestring): + commit = load_commit(commit) + + return commit.id[0:7]+"... " + commit.subject[0:59] + +# Return the directory name with .git stripped as a short identifier +# for the module +def get_module_name(): + try: + git_dir = git.rev_parse(git_dir=True, _quiet=True) + except CalledProcessError: + die("GIT_DIR not set") + + # Use the directory name with .git stripped as a short identifier + absdir = os.path.abspath(git_dir) + if absdir.endswith(os.sep + '.git'): + absdir = os.path.dirname(absdir) + projectshort = os.path.basename(absdir) + if projectshort.endswith(".git"): + projectshort = projectshort[:-4] + + return projectshort + +# Return the project description or '' if it is 'Unnamed repository;' +def get_project_description(): + try: + git_dir = git.rev_parse(git_dir=True, _quiet=True) + except CalledProcessError: + die("GIT_DIR not set") + + projectdesc = '' + description = os.path.join(git_dir, 'description') + if os.path.exists(description): + try: + projectdesc = open(description).read().strip() + except: + pass + if projectdesc.startswith('Unnamed repository;'): + projectdesc = '' + + return projectdesc diff --git a/roles/git/hooks/files/gnome-post-receive-email b/roles/git/hooks/files/gnome-post-receive-email new file mode 100644 index 0000000000..4c5cd619b6 --- /dev/null +++ b/roles/git/hooks/files/gnome-post-receive-email @@ -0,0 +1,941 @@ +#!/usr/bin/python +# +# gnome-post-receive-email - Post receive email hook for the GNOME Git repository +# +# Copyright (C) 2008 Owen Taylor +# Copyright (C) 2009 Red Hat, Inc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, If not, see +# http://www.gnu.org/licenses/. +# +# About +# ===== +# This script is used to generate mail to commits-list@gnome.org when change +# are pushed to the GNOME git repository. It accepts input in the form of +# a Git post-receive hook, and generates appropriate emails. +# +# The attempt here is to provide a maximimally useful and robust output +# with as little clutter as possible. +# + +import re +import os +import pwd +import sys +from email.header import Header +from socket import gethostname + +from kitchen.text.converters import to_bytes, to_unicode +from kitchen.text.misc import byte_string_valid_encoding + +script_path = os.path.realpath(os.path.abspath(sys.argv[0])) +script_dir = os.path.dirname(script_path) + +sys.path.insert(0, script_dir) + +from git import * +from util import die, strip_string as s, start_email, end_email + +# When we put a git subject into the Subject: line, where to truncate +SUBJECT_MAX_SUBJECT_CHARS = 100 + +CREATE = 0 +UPDATE = 1 +DELETE = 2 +INVALID_TAG = 3 + +# Short name for project +projectshort = None + +# Project description +projectdesc = None + +# Human readable name for user, might be None +user_fullname = None + +# Who gets the emails +recipients = None + +# What domain the emails are from +maildomain = None + +# short diff output only +mailshortdiff = False + +# map of ref_name => Change object; this is used when computing whether +# we've previously generated a detailed diff for a commit in the push +all_changes = {} +processed_changes = {} + +class RefChange(object): + def __init__(self, refname, oldrev, newrev): + self.refname = refname + self.oldrev = oldrev + self.newrev = newrev + + if oldrev == None and newrev != None: + self.change_type = CREATE + elif oldrev != None and newrev == None: + self.change_type = DELETE + elif oldrev != None and newrev != None: + self.change_type = UPDATE + else: + self.change_type = INVALID_TAG + + m = re.match(r"refs/[^/]*/(.*)", refname) + if m: + self.short_refname = m.group(1) + else: + self.short_refname = refname + + # Do any setup before sending email. The __init__ function should generally + # just record the parameters passed in and not do git work. (The main reason + # for the split is to let the prepare stage do different things based on + # whether other ref updates have been processed or not.) + def prepare(self): + pass + + # Whether we should generate the normal 'main' email. For simple branch + # updates we only generate 'extra' emails + def get_needs_main_email(self): + return True + + # The XXX in [projectname/XXX], usually a branch + def get_project_extra(self): + return None + + # Return the subject for the main email, without the leading [projectname] + def get_subject(self): + raise NotImplementedError() + + # Write the body of the main email to the given file object + def generate_body(self, out): + raise NotImplementedError() + + def generate_header(self, out, subject, include_revs=True, oldrev=None, newrev=None): + user = os.environ['USER'] + if user_fullname: + from_address = "%s <%s@%s>" % (user_fullname, user, maildomain) + else: + from_address = "%s@%s" % (user, maildomain) + + if not byte_string_valid_encoding(to_bytes(subject), 'ascii'): + # non-ascii chars + subject = Header(to_bytes(to_unicode(subject)), 'utf-8').encode() + + print >>out, s(""" +To: %(recipients)s +From: %(from_address)s +Subject: %(subject)s +MIME-Version: 1.0 +Content-Transfer-Encoding: 8bit +Content-Type: text/plain; charset="utf-8" +Keywords: %(projectshort)s +X-Project: %(projectdesc)s +X-Git-Refname: %(refname)s +""") % { + 'recipients': to_bytes(recipients, errors='strict'), + 'from_address': to_bytes(from_address, errors='strict'), + 'subject': subject, + 'projectshort': to_bytes(projectshort), + 'projectdesc': to_bytes(projectdesc), + 'refname': to_bytes(self.refname) + } + + if include_revs: + if oldrev: + oldrev = oldrev + else: + oldrev = NULL_REVISION + if newrev: + newrev = newrev + else: + newrev = NULL_REVISION + + print >>out, s(""" +X-Git-Oldrev: %(oldrev)s +X-Git-Newrev: %(newrev)s +""") % { + 'oldrev': to_bytes(oldrev), + 'newrev': to_bytes(newrev), + } + + # Trailing newline to signal the end of the header + print >>out + + def send_main_email(self): + if not self.get_needs_main_email(): + return + + extra = self.get_project_extra() + if extra: + extra = "/" + extra + else: + extra = "" + subject = "[" + projectshort + extra + "] " + self.get_subject() + + email_out = start_email() + + self.generate_header(email_out, subject, include_revs=True, oldrev=self.oldrev, newrev=self.newrev) + self.generate_body(email_out) + + end_email() + + # Allow multiple emails to be sent - used for branch updates + def send_extra_emails(self): + pass + + def send_emails(self): + self.send_main_email() + self.send_extra_emails() + +# ======================== + +# Common baseclass for BranchCreation and BranchUpdate (but not BranchDeletion) +class BranchChange(RefChange): + def __init__(self, *args): + RefChange.__init__(self, *args) + + def prepare(self): + # We need to figure out what commits are referenced in this commit thta + # weren't previously referenced in the repository by another branch. + # "Previously" here means either before this push, or by branch updates + # we've already done in this push. These are the commits we'll send + # out individual mails for. + # + # Note that "Before this push" can't be gotten exactly right since an + # push is only atomic per-branch and there is no locking across branches. + # But new commits will always show up in a cover mail in any case; even + # someone who maliciously is trying to fool us can't hide all trace. + + # Ordering matters here, so we can't rely on kwargs + branches = git.rev_parse('--symbolic-full-name', '--branches', _split_lines=True) + detailed_commit_args = [ self.newrev ] + + for branch in branches: + if branch == self.refname: + # For this branch, exclude commits before 'oldrev' + if self.change_type != CREATE: + detailed_commit_args.append("^" + self.oldrev) + elif branch in all_changes and not branch in processed_changes: + # For branches that were updated in this push but we haven't processed + # yet, exclude commits before their old revisions + detailed_commit_args.append("^" + all_changes[branch].oldrev) + else: + # Exclude commits that are ancestors of all other branches + detailed_commit_args.append("^" + branch) + + detailed_commits = git.rev_list(*detailed_commit_args).splitlines() + + self.detailed_commits = set() + for id in detailed_commits: + self.detailed_commits.add(id) + + # Find the commits that were added and removed, reverse() to get + # chronological order + if self.change_type == CREATE: + # If someone creates a branch of GTK+, we don't want to list (or even walk through) + # all 30,000 commits in the history as "new commits" on the branch. So we start + # the commit listing from the first commit we are going to send a mail out about. + # + # This does mean that if someone creates a branch, merges it, and then pushes + # both the branch and what was merged into at once, then the resulting mails will + # be a bit strange (depending on ordering) - the mail for the creation of the + # branch may look like it was created in the finished state because all the commits + # have been already mailed out for the other branch. I don't think this is a big + # problem, and the best way to fix it would be to sort the ref updates so that the + # branch creation was processed first. + # + if len(detailed_commits) > 0: + # Verify parent of first detailed commit is valid. On initial push, it is not. + parent = detailed_commits[-1] + "^" + try: + validref = git.rev_parse(parent, _quiet=True) + except CalledProcessError, error: + self.added_commits = [] + else: + self.added_commits = rev_list_commits(parent + ".." + self.newrev) + self.added_commits.reverse() + else: + self.added_commits = [] + self.removed_commits = [] + else: + self.added_commits = rev_list_commits(self.oldrev + ".." + self.newrev) + self.added_commits.reverse() + self.removed_commits = rev_list_commits(self.newrev + ".." + self.oldrev) + self.removed_commits.reverse() + + # In some cases we'll send a cover email that describes the overall + # change to the branch before ending individual mails for commits. In other + # cases, we just send the individual emails. We generate a cover mail: + # + # - If it's a branch creation + # - If it's not a fast forward + # - If there are any merge commits + # - If there are any commits we won't send separately (already in repo) + + have_merge_commits = False + for commit in self.added_commits: + if commit_is_merge(commit): + have_merge_commits = True + + self.needs_cover_email = (self.change_type == CREATE or + len(self.removed_commits) > 0 or + have_merge_commits or + len(self.detailed_commits) < len(self.added_commits)) + + def get_needs_main_email(self): + return self.needs_cover_email + + # A prefix for the cover letter summary with the number of added commits + def get_count_string(self): + if len(self.added_commits) > 1: + return "(%d commits) " % len(self.added_commits) + else: + return "" + + # Generate a short listing for a series of commits + # show_details - whether we should mark commit where we aren't going to send + # a detailed email. (Set the False when listing removed commits) + def generate_commit_summary(self, out, commits, show_details=True): + detail_note = False + for commit in commits: + if show_details and not commit.id in self.detailed_commits: + detail = " (*)" + detail_note = True + else: + detail = "" + print >>out, " %s%s" % (to_bytes(commit_oneline(commit)), to_bytes(detail)) + + if detail_note: + print >>out + print >>out, "(*) This commit already existed in another branch; no separate mail sent" + + def send_extra_emails(self): + total = len(self.added_commits) + + for i, commit in enumerate(self.added_commits): + if not commit.id in self.detailed_commits: + continue + + email_out = start_email() + + if self.short_refname == 'master': + branch = "" + else: + branch = "/" + self.short_refname + + total = len(self.added_commits) + if total > 1 and self.needs_cover_email: + count_string = ": %(index)s/%(total)s" % { + 'index' : i + 1, + 'total' : total + } + else: + count_string = "" + + subject = "[%(projectshort)s%(branch)s%(count_string)s] %(subject)s" % { + 'projectshort' : projectshort, + 'branch' : branch, + 'count_string' : count_string, + 'subject' : commit.subject[0:SUBJECT_MAX_SUBJECT_CHARS] + } + + # If there is a cover email, it has the X-Git-OldRev/X-Git-NewRev in it + # for the total branch update. Without a cover email, we are conceptually + # breaking up the update into individual updates for each commit + if self.needs_cover_email: + self.generate_header(email_out, subject, include_revs=False) + else: + parent = git.rev_parse(commit.id + "^") + self.generate_header(email_out, subject, + include_revs=True, + oldrev=parent, newrev=commit.id) + + email_out.flush() + git.show(commit.id, M=True, stat=True, _outfile=email_out) + email_out.flush() + if not mailshortdiff: + git.show(commit.id, p=True, M=True, diff_filter="ACMRTUXB", pretty="format:---", _outfile=email_out) + end_email() + +class BranchCreation(BranchChange): + def get_subject(self): + return self.get_count_string() + "Created branch " + self.short_refname + + def generate_body(self, out): + if len(self.added_commits) > 0: + print >>out, s(""" +The branch '%(short_refname)s' was created. + +Summary of new commits: + +""") % { + 'short_refname': to_bytes(self.short_refname), + } + + self.generate_commit_summary(out, self.added_commits) + else: + print >>out, s(""" +The branch '%(short_refname)s' was created pointing to: + + %(commit_oneline)s + +""") % { + 'short_refname': to_bytes(self.short_refname), + 'commit_oneline': to_bytes(commit_oneline(self.newrev)) + } + +class BranchUpdate(BranchChange): + def get_project_extra(self): + if len(self.removed_commits) > 0: + # In the non-fast-forward-case, the branch name is in the subject + return None + else: + if self.short_refname == 'master': + # Not saying 'master' all over the place reduces clutter + return None + else: + return self.short_refname + + def get_subject(self): + if len(self.removed_commits) > 0: + return self.get_count_string() + "Non-fast-forward update to branch " + self.short_refname + else: + # We want something for useful for the subject than "Updates to branch spiffy-stuff". + # The common case where we have a cover-letter for a fast-forward branch + # update is a merge. So we try to get: + # + # [myproject/spiffy-stuff] (18 commits) ...Merge branch master + # + last_commit = self.added_commits[-1] + if len(self.added_commits) > 1: + return self.get_count_string() + "..." + last_commit.subject[0:SUBJECT_MAX_SUBJECT_CHARS] + else: + # The ... indicates we are only showing one of many, don't need it for a single commit + return last_commit.subject[0:SUBJECT_MAX_SUBJECT_CHARS] + + def generate_body_normal(self, out): + print >>out, s(""" +Summary of changes: + +""") + + self.generate_commit_summary(out, self.added_commits) + + def generate_body_non_fast_forward(self, out): + print >>out, s(""" +The branch '%(short_refname)s' was changed in a way that was not a fast-forward update. +NOTE: This may cause problems for people pulling from the branch. For more information, +please see: + + http://live.gnome.org/Git/Help/NonFastForward + +Commits removed from the branch: + +""") % { + 'short_refname': to_bytes(self.short_refname), + } + + self.generate_commit_summary(out, self.removed_commits, show_details=False) + + print >>out, s(""" + +Commits added to the branch: + +""") + self.generate_commit_summary(out, self.added_commits) + + def generate_body(self, out): + if len(self.removed_commits) == 0: + self.generate_body_normal(out) + else: + self.generate_body_non_fast_forward(out) + +class BranchDeletion(RefChange): + def get_subject(self): + return "Deleted branch " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The branch '%(short_refname)s' was deleted. +""") % { + 'short_refname': to_bytes(self.short_refname), + } + +# ======================== + +class AnnotatedTagChange(RefChange): + def __init__(self, *args): + RefChange.__init__(self, *args) + + def prepare(self): + # Resolve tag to commit + if self.oldrev: + self.old_commit_id = git.rev_parse(self.oldrev + "^{commit}") + + if self.newrev: + self.parse_tag_object(self.newrev) + else: + self.parse_tag_object(self.oldrev) + + # Parse information out of the tag object + def parse_tag_object(self, revision): + message_lines = [] + in_message = False + + # A bit of paranoia if we fail at parsing; better to make the failure + # visible than just silently skip Tagger:/Date:. + self.tagger = "unknown " + self.date = "at an unknown time" + + self.have_signature = False + for line in git.cat_file(revision, p=True, _split_lines=True): + if in_message: + # Nobody is going to verify the signature by extracting it + # from the email, so strip it, and remember that we saw it + # by saying 'signed tag' + if re.match(r'-----BEGIN PGP SIGNATURE-----', line): + self.have_signature = True + break + message_lines.append(line) + else: + if line.strip() == "": + in_message = True + continue + # I don't know what a more robust rule is for dividing the + # name and date, other than maybe looking explicitly for a + # RFC 822 date. This seems to work pretty well + m = re.match(r"tagger\s+([^>]*>)\s*(.*)", line) + if m: + self.tagger = m.group(1) + self.date = m.group(2) + continue + self.message = "\n".join([" " + line for line in message_lines]) + + # Outputs information about the new tag + def generate_tag_info(self, out): + + print >>out, s(""" +Tagger: %(tagger)s +Date: %(date)s + +%(message)s + +""") % { + 'tagger': to_bytes(self.tagger), + 'date': to_bytes(self.date), + 'message': to_bytes(self.message), + } + + # We take the creation of an annotated tag as being a "mini-release-announcement" + # and show a 'git shortlog' of the changes since the last tag that was an + # ancestor of the new tag. + last_tag = None + try: + # A bit of a hack to get that previous tag + last_tag = git.describe(self.newrev+"^", abbrev='0', _quiet=True) + except CalledProcessError: + # Assume that this means no older tag + pass + + if last_tag: + revision_range = last_tag + ".." + self.newrev + print >>out, s(""" +Changes since the last tag '%(last_tag)s': + +""") % { + 'last_tag': to_bytes(last_tag) + } + else: + revision_range = self.newrev + print >>out, s(""" +Changes: + +""") + out.write(to_bytes(git.shortlog(revision_range))) + out.write("\n") + + def get_tag_type(self): + if self.have_signature: + return 'signed tag' + else: + return 'unsigned tag' + +class AnnotatedTagCreation(AnnotatedTagChange): + def get_subject(self): + return "Created tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The %(tag_type)s '%(short_refname)s' was created. + +""") % { + 'tag_type': to_bytes(self.get_tag_type()), + 'short_refname': to_bytes(self.short_refname), + } + self.generate_tag_info(out) + +class AnnotatedTagDeletion(AnnotatedTagChange): + def get_subject(self): + return "Deleted tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The %(tag_type)s '%(short_refname)s' was deleted. It previously pointed to: + + %(old_commit_oneline)s +""") % { + 'tag_type': to_bytes(self.get_tag_type()), + 'short_refname': to_bytes(self.short_refname), + 'old_commit_oneline': to_bytes(commit_oneline(self.old_commit_id)), + } + +class AnnotatedTagUpdate(AnnotatedTagChange): + def get_subject(self): + return "Updated tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The tag '%(short_refname)s' was replaced with a new tag. It previously +pointed to: + + %(old_commit_oneline)s + +NOTE: People pulling from the repository will not get the new tag. +For more information, please see: + + http://live.gnome.org/Git/Help/TagUpdates + +New tag information: + +""") % { + 'short_refname': to_bytes(self.short_refname), + 'old_commit_oneline': to_bytes(commit_oneline(self.old_commit_id)), + } + self.generate_tag_info(out) + +# ======================== + +class LightweightTagCreation(RefChange): + def get_subject(self): + return "Created tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The lightweight tag '%(short_refname)s' was created pointing to: + + %(commit_oneline)s +""") % { + 'short_refname': to_bytes(self.short_refname), + 'commit_oneline': to_bytes(commit_oneline(self.newrev)) + } + +class LightweightTagDeletion(RefChange): + def get_subject(self): + return "Deleted tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The lighweight tag '%(short_refname)s' was deleted. It previously pointed to: + + %(commit_oneline)s +""") % { + 'short_refname': to_bytes(self.short_refname), + 'commit_oneline': to_bytes(commit_oneline(self.oldrev)), + } + +class LightweightTagUpdate(RefChange): + def get_subject(self): + return "Updated tag " + self.short_refname + + def generate_body(self, out): + print >>out, s(""" +The lightweight tag '%(short_refname)s' was updated to point to: + + %(commit_oneline)s + +It previously pointed to: + + %(old_commit_oneline)s + +NOTE: People pulling from the repository will not get the new tag. +For more information, please see: + + http://live.gnome.org/Git/Help/TagUpdates +""") % { + 'short_refname': to_bytes(self.short_refname), + 'commit_oneline': to_bytes(commit_oneline(self.newrev)), + 'old_commit_oneline': to_bytes(commit_oneline(self.oldrev)), + } + +# ======================== + +class InvalidRefDeletion(RefChange): + def get_subject(self): + return "Deleted invalid ref " + self.refname + + def generate_body(self, out): + print >>out, s(""" +The ref '%(refname)s' was deleted. It previously pointed nowhere. +""") % { + 'refname': to_bytes(self.refname), + } + +# ======================== + +class MiscChange(RefChange): + def __init__(self, refname, oldrev, newrev, message): + RefChange.__init__(self, refname, oldrev, newrev) + self.message = message + +class MiscCreation(MiscChange): + def get_subject(self): + return "Unexpected: Created " + self.refname + + def generate_body(self, out): + print >>out, s(""" +The ref '%(refname)s' was created pointing to: + + %(newrev)s + +This is unexpected because: + + %(message)s +""") % { + 'refname': to_bytes(self.refname), + 'newrev': to_bytes(self.newrev), + 'message': to_bytes(self.message), + } + +class MiscDeletion(MiscChange): + def get_subject(self): + return "Unexpected: Deleted " + self.refname + + def generate_body(self, out): + print >>out, s(""" +The ref '%(refname)s' was deleted. It previously pointed to: + + %(oldrev)s + +This is unexpected because: + + %(message)s +""") % { + 'refname': to_bytes(self.refname), + 'oldrev': to_bytes(self.oldrev), + 'message': to_bytes(self.message), + } + +class MiscUpdate(MiscChange): + def get_subject(self): + return "Unexpected: Updated " + self.refname + + def generate_body(self, out): + print >>out, s(""" +The ref '%(refname)s' was updated from: + + %(newrev)s + +To: + + %(oldrev)s + +This is unexpected because: + + %(message)s +""") % { + 'refname': to_bytes(self.refname), + 'oldrev': to_bytes(self.oldrev), + 'newrev': to_bytes(self.newrev), + 'message': to_bytes(self.message), + } + +# ======================== + +def make_change(oldrev, newrev, refname): + refname = refname + + # Canonicalize + oldrev = git.rev_parse(oldrev) + newrev = git.rev_parse(newrev) + + # Replacing the null revision with None makes it easier for us to test + # in subsequent code + + if re.match(r'^0+$', oldrev): + oldrev = None + else: + oldrev = oldrev + + if re.match(r'^0+$', newrev): + newrev = None + else: + newrev = newrev + + # Figure out what we are doing to the ref + + if oldrev == None and newrev != None: + change_type = CREATE + target = newrev + elif oldrev != None and newrev == None: + change_type = DELETE + target = oldrev + elif oldrev != None and newrev != None: + change_type = UPDATE + target = newrev + else: + return InvalidRefDeletion(refname, oldrev, newrev) + + object_type = git.cat_file(target, t=True) + + # And then create the right type of change object + + # Closing the arguments like this simplifies the following code + def make(cls, *args): + return cls(refname, oldrev, newrev, *args) + + def make_misc_change(message): + if change_type == CREATE: + return make(MiscCreation, message) + elif change_type == DELETE: + return make(MiscDeletion, message) + else: + return make(MiscUpdate, message) + + if re.match(r'^refs/tags/.*$', refname): + if object_type == 'commit': + if change_type == CREATE: + return make(LightweightTagCreation) + elif change_type == DELETE: + return make(LightweightTagDeletion) + else: + return make(LightweightTagUpdate) + elif object_type == 'tag': + if change_type == CREATE: + return make(AnnotatedTagCreation) + elif change_type == DELETE: + return make(AnnotatedTagDeletion) + else: + return make(AnnotatedTagUpdate) + else: + return make_misc_change("%s is not a commit or tag object" % target) + elif re.match(r'^refs/heads/.*$', refname): + if object_type == 'commit': + if change_type == CREATE: + return make(BranchCreation) + elif change_type == DELETE: + return make(BranchDeletion) + else: + return make(BranchUpdate) + else: + return make_misc_change("%s is not a commit object" % target) + elif re.match(r'^refs/remotes/.*$', refname): + return make_misc_change("'%s' is a tracking branch and doesn't belong on the server" % refname) + else: + return make_misc_change("'%s' is not in refs/heads/ or refs/tags/" % refname) + +def main(): + global projectshort + global projectdesc + global user_fullname + global recipients + global maildomain + global mailshortdiff + + # No emails for a repository in the process of being imported + git_dir = git.rev_parse(git_dir=True, _quiet=True) + if os.path.exists(os.path.join(git_dir, 'pending')): + return + + projectshort = get_module_name() + projectdesc = get_project_description() + + + try: + mailshortdiff=git.config("hooks.mailshortdiff", _quiet=True) + except CalledProcessError: + pass + + if isinstance(mailshortdiff, str) and mailshortdiff.lower() in ('true', 'yes', 'on', '1'): + mailshortdiff = True + else: + mailshortdiff = False + + try: + recipients=git.config("hooks.mailinglist", _quiet=True) + except CalledProcessError: + pass + + if not recipients: + die("hooks.mailinglist is not set") + + # Get the domain name to use in the From header + try: + maildomain = git.config("hooks.maildomain", _quiet=True) + except CalledProcessError: + pass + + if not maildomain: + try: + hostname = gethostname() + maildomain = '.'.join(hostname.split('.')[1:]) + except: + pass + if not maildomain or '.' not in maildomain: + maildomain = 'localhost.localdomain' + + # Figure out a human-readable username + try: + entry = pwd.getpwuid(os.getuid()) + gecos = entry.pw_gecos + except: + gecos = None + + if gecos != None: + # Typical GNOME account have John Doe for the GECOS. + # Comma-separated fields are also possible + m = re.match("([^,<]+)", gecos) + if m: + fullname = m.group(1).strip() + if fullname != "": + try: + user_fullname = unicode(fullname, 'ascii') + except UnicodeDecodeError: + user_fullname = Header(fullname, 'utf-8').encode() + + changes = [] + + if len(sys.argv) > 1: + # For testing purposes, allow passing in a ref update on the command line + if len(sys.argv) != 4: + die("Usage: generate-commit-mail OLDREV NEWREV REFNAME") + changes.append(make_change(sys.argv[1], sys.argv[2], sys.argv[3])) + else: + for line in sys.stdin: + items = line.strip().split() + if len(items) != 3: + die("Input line has unexpected number of items") + changes.append(make_change(items[0], items[1], items[2])) + + for change in changes: + all_changes[change.refname] = change + + for change in changes: + change.prepare() + change.send_emails() + processed_changes[change.refname] = change + +if __name__ == '__main__': + main() diff --git a/roles/git/hooks/files/post-received-chained b/roles/git/hooks/files/post-received-chained new file mode 100644 index 0000000000..b5c6e23112 --- /dev/null +++ b/roles/git/hooks/files/post-received-chained @@ -0,0 +1,8 @@ +#!/bin/bash +# Redirect stdin to each of the post-receive hooks in place. + +# You need to explicitly add your hook to the following list +# for it to be invoked. +pee \ + $GIT_DIR/hooks/post-receive-chained.d/post-receive-email \ + $GIT_DIR/hooks/post-receive-chained.d/post-receive-fedmsg diff --git a/roles/git/hooks/files/post-received-fedmsg b/roles/git/hooks/files/post-received-fedmsg new file mode 100644 index 0000000000..7bc9a140d9 --- /dev/null +++ b/roles/git/hooks/files/post-received-fedmsg @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import getpass +import git +import os +import sys + +import fedmsg +import fedmsg.config + +# Read in all the rev information git-receive-pack hands us. +lines = [line.split() for line in sys.stdin.readlines()] + +# Use $GIT_DIR to determine where this repo is. +abspath = os.path.abspath(os.environ['GIT_DIR']) +repo_name = '.'.join(abspath.split(os.path.sep)[-1].split('.')[:-1]) + +username = getpass.getuser() + +repo = git.repo.Repo(abspath) +def _build_commit(rev): + old, rev, branch = rev + branch = '/'.join(branch.split('/')[2:]) + commit = repo.rev_parse(rev=rev) + + # We just don't handle these + if isinstance(commit, git.TagObject): + return None + + return dict( + name=commit.author.name, + email=commit.author.email, + username=username, + summary=commit.summary, + message=commit.message, + stats=dict( + files=commit.stats.files, + total=commit.stats.total, + ), + rev=rev, + path=abspath, + repo=repo_name, + branch=branch, + agent=os.getlogin(), + ) + +commits = map(_build_commit, lines) + +print "Emitting a message to the fedmsg bus." +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='scm', **config) + +for commit in commits: + + if commit is None: + continue + + fedmsg.publish( + # Expect this to change to just "receive" in the future. + topic="receive", + msg=dict(commit=commit), + modname="git", + ) diff --git a/roles/git/hooks/files/util.py b/roles/git/hooks/files/util.py new file mode 100644 index 0000000000..f35019634f --- /dev/null +++ b/roles/git/hooks/files/util.py @@ -0,0 +1,153 @@ +# General Utility Functions used in our Git scripts +# +# Copyright (C) 2008 Owen Taylor +# Copyright (C) 2009 Red Hat, Inc +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, If not, see +# http://www.gnu.org/licenses/. + +import os +import sys +from subprocess import Popen +import tempfile +import time + +def die(message): + print >>sys.stderr, message + sys.exit(1) + +# This cleans up our generation code by allowing us to use the same indentation +# for the first line and subsequent line of a multi-line string +def strip_string(str): + start = 0 + end = len(str) + if len(str) > 0 and str[0] == '\n': + start += 1 + if len(str) > 1 and str[end - 1] == '\n': + end -= 1 + + return str[start:end] + +# How long to wait between mails (in seconds); the idea of waiting +# is to try to make the sequence of mails we send out in order +# actually get delivered in order. The waiting is done in a forked +# subprocess and doesn't stall completion of the main script. +EMAIL_DELAY = 5 + +# Some line that can never appear in any email we send out +EMAIL_BOUNDARY="---@@@--- gnome-git-email ---@@@---\n" + +# Run in subprocess +def _do_send_emails(email_in): + email_files = [] + current_file = None + last_line = None + + # Read emails from the input pipe and write each to a file + for line in email_in: + if current_file is None: + current_file, filename = tempfile.mkstemp(suffix=".mail", prefix="gnome-post-receive-email-") + email_files.append(filename) + + if line == EMAIL_BOUNDARY: + # Strip the last line if blank; see comment when writing + # the email boundary for rationale + if last_line.strip() != "": + os.write(current_file, last_line) + last_line = None + os.close(current_file) + current_file = None + else: + if last_line is not None: + os.write(current_file, last_line) + last_line = line + + if current_file is not None: + if last_line is not None: + os.write(current_file, last_line) + os.close(current_file) + + # We're done interacting with the parent process, the rest happens + # asynchronously; send out the emails one by one and remove the + # temporary files + for i, filename in enumerate(email_files): + if i != 0: + time.sleep(EMAIL_DELAY) + + f = open(filename, "r") + process = Popen(["/usr/sbin/sendmail", "-t"], + stdout=None, stderr=None, stdin=f) + process.wait() + f.close() + + os.remove(filename) + +email_file = None + +# Start a new outgoing email; returns a file object that the +# email should be written to. Call end_email() when done +def start_email(): + global email_file + if email_file is None: + email_pipe = os.pipe() + pid = os.fork() + if pid == 0: + # The child + + os.close(email_pipe[1]) + email_in = os.fdopen(email_pipe[0]) + + # Redirect stdin/stdout/stderr to/from /dev/null + devnullin = os.open("/dev/null", os.O_RDONLY) + os.close(0) + os.dup2(devnullin, 0) + + devnullout = os.open("/dev/null", os.O_WRONLY) + os.close(1) + os.dup2(devnullout, 1) + os.close(2) + os.dup2(devnullout, 2) + os.close(devnullout) + + # Fork again to daemonize + if os.fork() > 0: + sys.exit(0) + + try: + _do_send_emails(email_in) + except Exception: + import syslog + import traceback + + syslog.openlog(os.path.basename(sys.argv[0])) + syslog.syslog(syslog.LOG_ERR, "Unexpected exception sending mail") + for line in traceback.format_exc().strip().split("\n"): + syslog.syslog(syslog.LOG_ERR, line) + + sys.exit(0) + + email_file = os.fdopen(email_pipe[1], "w") + else: + # The email might not end with a newline, so add one. We'll + # strip the last line, if blank, when emails, so the net effect + # is to add a newline to messages without one + email_file.write("\n") + email_file.write(EMAIL_BOUNDARY) + + return email_file + +# Finish an email started with start_email +def end_email(): + global email_file + email_file.flush() diff --git a/roles/git/hooks/tasks/main.yml b/roles/git/hooks/tasks/main.yml new file mode 100644 index 0000000000..6cf6b9d0f3 --- /dev/null +++ b/roles/git/hooks/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# tasklist for setting up git mail hooks + +- name: install needed packages + yum: pkg={{item}} state=present + with_items: + - git + - moreutils + +# This requires the fedmsg/base role +- name: install the git hooks + copy: src={{item}} dest=/usr/share/git-core mode=0755 + with_items: + - post-receive-fedmsg + - post-receive-chained + +- name: install the git mail hooks + copy: src={{item}} dest=/usr/share/git-core/mail-hooks mode=0755 + with_items: + - util.py + - git.py + - gnome-post-receive-email From 3cbdbf5a76d69e2d7268d066668a834340157ab0 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Mon, 7 Jul 2014 12:28:28 +0200 Subject: [PATCH 116/193] Add a new git/server role This will be needed to migrate Dist Git from puppet to ansible. --- roles/git/server/tasks/main.yml | 16 ++++++++++++++++ roles/git/server/templates/git.j2 | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 roles/git/server/tasks/main.yml create mode 100644 roles/git/server/templates/git.j2 diff --git a/roles/git/server/tasks/main.yml b/roles/git/server/tasks/main.yml new file mode 100644 index 0000000000..e8aea839a3 --- /dev/null +++ b/roles/git/server/tasks/main.yml @@ -0,0 +1,16 @@ +--- +# tasklist for setting up a git server (git:// access) + +- name: install the needed packages + yum: pkg={{item}} state=present + with_items: + - git-daemon + - xinetd + +- name: install the xinetd config file + template: > + src="git.j2" + dest="/etc/xinetd.d/git" + mode=0644 + notify: + - restart xinetd diff --git a/roles/git/server/templates/git.j2 b/roles/git/server/templates/git.j2 new file mode 100644 index 0000000000..5c08297ef8 --- /dev/null +++ b/roles/git/server/templates/git.j2 @@ -0,0 +1,16 @@ +# default: off +# description: The git server offers access to git repositories +service git +{ + disable = no + type = UNLISTED + port = {{ git_port }} + socket_type = stream + wait = no + groups = yes + group = {{ git_group }} + user = nobody + server = {{ git_server }} + server_args = {{ git_server_args }} --base-path={{ git_basepath }} --base-path-relaxed + log_on_failure += HOST +} From 360fb1fa335534aed2d290acc1b83489e64fe7a4 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 8 Jul 2014 11:52:29 +0200 Subject: [PATCH 117/193] Add a new gitolite/base role This will be needed to migrate Dist Git from puppet to ansible. --- roles/gitolite/base/tasks/main.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 roles/gitolite/base/tasks/main.yml diff --git a/roles/gitolite/base/tasks/main.yml b/roles/gitolite/base/tasks/main.yml new file mode 100644 index 0000000000..b6c7916078 --- /dev/null +++ b/roles/gitolite/base/tasks/main.yml @@ -0,0 +1,10 @@ +--- +# tasklist for setting up a basic gitolite + +- name: install the needed packages + yum: pkg={{item}} state=present + with_items: + - gitolite + +- name: ensure the /etc/gitolite directory + file: path=/etc/gitolite owner=root group=root mode=0755 state=directory From be0f5446f08ce3e65a64a509467365682bee9b95 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 8 Jul 2014 15:28:05 +0200 Subject: [PATCH 118/193] Add a new cgit/base role This will be needed to migrate Dist Git from puppet to ansible. --- roles/cgit/base/files/cgit-fedora.css | 664 ++++++++++++++++++ roles/cgit/base/files/cgit.css | 253 +++++++ roles/cgit/base/files/cgit.png | Bin 0 -> 16124 bytes .../cgit/base/files/email-libravatar-korg.lua | 15 + roles/cgit/base/files/favicon.ico | Bin 0 -> 3638 bytes roles/cgit/base/files/fedora-layout.css | 300 ++++++++ roles/cgit/base/files/fedora-style.css | 225 ++++++ roles/cgit/base/files/html-bg.png | Bin 0 -> 312 bytes roles/cgit/base/files/t.png | Bin 0 -> 317 bytes roles/cgit/base/tasks/main.yml | 26 + 10 files changed, 1483 insertions(+) create mode 100644 roles/cgit/base/files/cgit-fedora.css create mode 100644 roles/cgit/base/files/cgit.css create mode 100644 roles/cgit/base/files/cgit.png create mode 100644 roles/cgit/base/files/email-libravatar-korg.lua create mode 100644 roles/cgit/base/files/favicon.ico create mode 100644 roles/cgit/base/files/fedora-layout.css create mode 100644 roles/cgit/base/files/fedora-style.css create mode 100644 roles/cgit/base/files/html-bg.png create mode 100644 roles/cgit/base/files/t.png create mode 100644 roles/cgit/base/tasks/main.yml diff --git a/roles/cgit/base/files/cgit-fedora.css b/roles/cgit/base/files/cgit-fedora.css new file mode 100644 index 0000000000..f36a619813 --- /dev/null +++ b/roles/cgit/base/files/cgit-fedora.css @@ -0,0 +1,664 @@ +body, table, form { + padding: 0em; + margin: 0em; +} + +a { + color: blue; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +table { + background: none; + border-collapse: collapse; +} + +table#header { + width: 100%; + margin-bottom: 1em; +} + +table#header td.logo { + width: 96px; +} + +table#header td.main { + font-size: 250%; + margin-top: 10px; + white-space: nowrap; + padding-left: 50px; +} + +table#header td.main a { + color: #000; +} + +table#header td.form { + text-align: right; + vertical-align: bottom; + padding-right: 1em; + padding-bottom: 2px; + white-space: nowrap; +} + +table#header td.form form, +table#header td.form input, +table#header td.form select { + font-size: 90%; +} + +table#header td.sub { + color: #777; + border-top: solid 1px #ccc; + padding-left: 60px; +} + +table.tabs { + /* border-bottom: solid 2px #ccc; */ + border-collapse: collapse; + margin-top: 2em; + margin-bottom: 0px; + width: 100%; +} + +table.tabs td { + padding: 0px 1em; + vertical-align: bottom; +} + +table.tabs td a { + padding: 2px 0.75em; + color: #777; + font-size: 110%; +} + +table.tabs td a.active { + color: #000; + background-color: #ccc; +} + +table.tabs td.form { + text-align: right; +} + +table.tabs td.form form { + padding-bottom: 2px; + font-size: 90%; + white-space: nowrap; +} + +table.tabs td.form input, +table.tabs td.form select { + font-size: 90%; +} + +div.path { + margin: 0px; + padding: 5px 2em 2px 2em; + color: #000; + background-color: #eee; +} + +div.content { + margin: 0px; + padding-top: 2em; + border-top: solid 3px #ccc; +} + +table.list { + width: 100%; + border: none; + border-collapse: collapse; +} + +table.list tr { + background: none; +} + +table.list tr.logheader { + background: #eee; +} + +table.list tr:hover { + background: #eee; +} + +table.list tr.nohover:hover { + background: white; +} + +table.list th { + font-weight: bold; + /* color: #888; + border-top: dashed 1px #888; + border-bottom: dashed 1px #888; + */ + padding: 0.1em 0.5em 0.05em 0.5em; + vertical-align: baseline; +} + +table.list td { + border: none; + padding: 0.1em 0.5em 0.1em 0.5em; +} + +table.list td.commitgraph { + font-family: monospace; + white-space: pre; +} + +table.list td.commitgraph .column1 { + color: #a00; +} + +table.list td.commitgraph .column2 { + color: #0a0; +} + +table.list td.commitgraph .column3 { + color: #aa0; +} + +table.list td.commitgraph .column4 { + color: #00a; +} + +table.list td.commitgraph .column5 { + color: #a0a; +} + +table.list td.commitgraph .column6 { + color: #0aa; +} + +table.list td.logsubject { + font-family: monospace; + font-weight: bold; +} + +table.list td.logmsg { + font-family: monospace; + white-space: pre; + padding: 0 0.5em; +} + +table.list td a { + color: black; +} + +table.list td a.ls-dir { + font-weight: bold; + color: #00f; +} + +table.list td a:hover { + color: #00f; +} + +img { + border: none; +} + +input#switch-btn { + margin: 2px 0px 0px 0px; +} + +td#sidebar input.txt { + width: 100%; + margin: 2px 0px 0px 0px; +} + +table#grid { + margin: 0px; +} + +td#content { + vertical-align: top; + padding: 1em 2em 1em 1em; + border: none; +} + +div#summary { + vertical-align: top; + margin-bottom: 1em; +} + +table#downloads { + float: right; + border-collapse: collapse; + border: solid 1px #777; + margin-left: 0.5em; + margin-bottom: 0.5em; +} + +table#downloads th { + background-color: #ccc; +} + +div#blob { + border: solid 1px black; +} + +div.error { + color: red; + font-weight: bold; + margin: 1em 2em; +} + +a.ls-blob, a.ls-dir, a.ls-mod { + font-family: monospace; +} + +td.ls-size { + text-align: right; + font-family: monospace; + width: 10em; +} + +td.ls-mode { + font-family: monospace; + width: 10em; +} + +table.blob { + margin-top: 0.5em; + border-top: solid 1px black; +} + +table.blob td.lines { + margin: 0; padding: 0 0 0 0.5em; + vertical-align: top; + color: black; +} + +table.blob td.linenumbers { + margin: 0; padding: 0 0.5em 0 0.5em; + vertical-align: top; + text-align: right; + border-right: 1px solid gray; + background-color: #eee; +} + +table.blob pre { + padding: 0; margin: 0; +} + +table.blob a.no, table.ssdiff a.no { + color: gray; + text-align: right; + text-decoration: none; +} + +table.blob a.no a:hover { + color: black; +} + +table.bin-blob { + margin-top: 0.5em; + border: solid 1px black; +} + +table.bin-blob th { + font-family: monospace; + white-space: pre; + border: solid 1px #777; + padding: 0.5em 1em; +} + +table.bin-blob td { + font-family: monospace; + white-space: pre; + border-left: solid 1px #777; + padding: 0em 1em; +} + +table.nowrap td { + white-space: nowrap; +} + +table.commit-info { + border-collapse: collapse; + margin-top: 1.5em; +} + +table.commit-info th { + text-align: left; + font-weight: normal; + padding: 0.1em 1em 0.1em 0.1em; + vertical-align: top; +} + +table.commit-info td { + font-weight: normal; + padding: 0.1em 1em 0.1em 0.1em; +} + +div.commit-subject { + font-weight: bold; + font-size: 125%; + margin: 1.5em 0em 0.5em 0em; + padding: 0em; +} + +div.commit-msg { + white-space: pre; + font-family: monospace; +} + +div.notes-header { + font-weight: bold; + padding-top: 1.5em; +} + +div.notes { + white-space: pre; + font-family: monospace; + border: solid 1px #ee9; + background-color: #ffd; + padding: 0.3em 2em 0.3em 1em; + float: left; +} + +div.notes-footer { + clear: left; +} + +div.diffstat-header { + font-weight: bold; + padding-top: 1.5em; +} + +table.diffstat { + border-collapse: collapse; + border: solid 1px #aaa; + background-color: #eee; +} + +table.diffstat th { + font-weight: normal; + text-align: left; + text-decoration: underline; + padding: 0.1em 1em 0.1em 0.1em; + font-size: 100%; +} + +table.diffstat td { + padding: 0.2em 0.2em 0.1em 0.1em; + font-size: 100%; + border: none; +} + +table.diffstat td.mode { + white-space: nowrap; +} + +table.diffstat td span.modechange { + padding-left: 1em; + color: red; +} + +table.diffstat td.add a { + color: green; +} + +table.diffstat td.del a { + color: red; +} + +table.diffstat td.upd a { + color: blue; +} + +table.diffstat td.graph { + width: 500px; + vertical-align: middle; +} + +table.diffstat td.graph table { + border: none; +} + +table.diffstat td.graph td { + padding: 0px; + border: 0px; + height: 7pt; +} + +table.diffstat td.graph td.add { + background-color: #5c5; +} + +table.diffstat td.graph td.rem { + background-color: #c55; +} + +div.diffstat-summary { + color: #888; + padding-top: 0.5em; +} + +table.diff { + width: 100%; +} + +table.diff td { + font-family: monospace; + white-space: pre; + font-size: 12px; +} + +table.diff td div.head { + font-weight: bold; + margin-top: 1em; + color: black; +} + +table.diff td div.hunk { + color: #009; +} + +table.diff td div.add { + color: green; +} + +table.diff td div.del { + color: red; +} + +.sha1 { + font-family: monospace; + font-size: 90%; +} + +.left { + text-align: left; +} + +.right { + text-align: right; +} + +table.list td.reposection { + font-style: italic; + color: #888; +} + +a.button { + font-size: 80%; + padding: 0em 0.5em; +} + +a.primary { + font-size: 100%; +} + +a.secondary { + font-size: 90%; +} + +td.toplevel-repo { + +} + +table.list td.sublevel-repo { + padding-left: 1.5em; +} + +div.pager { + text-align: center; + margin: 1em 0em 0em 0em; +} + +div.pager a { + color: #777; + margin: 0em 0.5em; +} + +span.age-mins { + font-weight: bold; + color: #080; +} + +span.age-hours { + color: #080; +} + +span.age-days { + color: #040; +} + +span.age-weeks { + color: #444; +} + +span.age-months { + color: #888; +} + +span.age-years { + color: #bbb; +} +div.footer { + margin-top: 0.5em; + text-align: center; + font-size: 80%; + color: #ccc; +} +a.branch-deco { + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #88ff88; + border: solid 1px #007700; +} +a.tag-deco { + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ffff88; + border: solid 1px #777700; +} +a.remote-deco { + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ccccff; + border: solid 1px #000077; +} +a.deco { + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ff8888; + border: solid 1px #770000; +} + +div.commit-subject a { + margin-left: 1em; + font-size: 75%; +} + +table.stats { + border: solid 1px black; + border-collapse: collapse; +} + +table.stats th { + text-align: left; + padding: 1px 0.5em; + background-color: #eee; + border: solid 1px black; +} + +table.stats td { + text-align: right; + padding: 1px 0.5em; + border: solid 1px black; +} + +table.stats td.total { + font-weight: bold; + text-align: left; +} + +table.stats td.sum { + color: #c00; + font-weight: bold; +/* background-color: #eee; */ +} + +table.stats td.left { + text-align: left; +} + +table.vgraph { + border-collapse: separate; + border: solid 1px black; + height: 200px; +} + +table.vgraph th { + background-color: #eee; + font-weight: bold; + border: solid 1px white; + padding: 1px 0.5em; +} + +table.vgraph td { + vertical-align: bottom; + padding: 0px 10px; +} + +table.vgraph div.bar { + background-color: #eee; +} + +table.hgraph { + border: solid 1px black; + width: 800px; +} + +table.hgraph th { + background-color: #eee; + font-weight: bold; + border: solid 1px black; + padding: 1px 0.5em; +} + +table.hgraph td { + vertical-align: center; + padding: 2px 2px; +} + +table.hgraph div.bar { + background-color: #eee; + height: 1em; +} + +table.ssdiff { + width: 100%; +} + +tbody { + width: 100%; +} diff --git a/roles/cgit/base/files/cgit.css b/roles/cgit/base/files/cgit.css new file mode 100644 index 0000000000..4a613651b0 --- /dev/null +++ b/roles/cgit/base/files/cgit.css @@ -0,0 +1,253 @@ +/* First include the Fedora style sheets. */ +@import "fedora-layout.css"; +@import "fedora-style.css"; +/* Then include the cgit style sheet */ +@import "cgit-fedora.css"; + +div#cgit span.libravatar img.onhover { + display: none; + border: 1px solid gray; + padding: 0px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + width: 128px; + height: 128px; +} + +div#cgit span.libravatar img.inline { + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + width: 13px; + height: 13px; + margin-right: 0.2em; + opacity: 0.4; +} + +div#cgit span.libravatar:hover > img.onhover { + display: block; + position: absolute; + margin-left: 1.5em; + background-color: #eeeeee; + box-shadow: 5px 5px 3px #bbb; +} + +table#header td.logo { + height: 72px; + width: 274px; +} + +/* Removing padding around body */ +body { + padding: 0px; + background: url(/cgit-data/images/html-bg.png) repeat-x scroll 0 10px #FFFFFF; + height: 100%; + margin-bottom: 0px; + color: #2E3436; + font-family: Cantarell,'Droid Sans','DejaVu Sans',Arial,sans-serif; + font-size: 13px; + line-height: 1.5; + width: 100%; +} + +#page { + width: 100%; +} + +/* Make link colors more gnome-ish */ +a, a:visited, a:hover, table.list td a:hover { + color: #3465A4; +} + +/* global domain bar */ +#global_domain_bar .maxwidth { + position: relative; +} + +.maxwidth { + margin: 0 auto; + width: 98%; +} + +#global_domain_bar .tab { + background: none repeat scroll 0 0 #FFFFFF; + border-radius: 5px 5px 5px 5px; + box-shadow: 0 4px 2px -2px #8FB3D9; + float: right; + font-size: 9px; + line-height: 16px; + padding: 4px; + margin-top: -17px; + right: 0; + white-space: nowrap; +} + +#global_domain_bar .tab a.root:last-child { + border-right: 0 none; + margin-right: 0; + padding-right: 0; +} + +#global_domain_bar .tab a.root { + background: url(cgit-data/images/favicon.png) no-repeat scroll 0 0 #FFFFFF; + border-right: 1px solid #CCCCCC; + color: #555753; + font-weight: bold; + margin-right: 3px; + padding-left: 18px; + padding-right: 6px; + text-decoration: none; +} + + +/* Kill some space in the cgit header, and mark it clear */ +table#header { + margin-top: 3em; + margin-bottom: 2em; + clear: both; +} + +/* Drop the font size for the heading down */ +table#header td.main { + font-size: 250%; +} + +/* When we killed the body padding, this cell is at the screen edge, move it back */ +table#header td.right { + padding-right: 4px; +} + +/* kill some more vertical space in the cgit header */ +table.tabs { + margin-top: 0em; +} + +/* FOOTER */ + +#footer { + background: none repeat scroll 0 0 #D3D7CF; + clear: left; + color: #555753; + padding: 14px 0 230px; + margin: auto; + font-size: 11px; + line-height: 1.5em; +} + +/* Make the branch/tag decoration a little smaller */ +a.branch-deco, a.tag-deco { + display: inline-block; + font-size: 90%; + padding: 1px 0.25em; +} + +/*top bar*/ + +#top_bar ul { + list-style: none outside none; + margin: 0; + padding: 0 5px; +} + +#top_bar ul li { + display: inline; + margin: 0 5px 0 0; +} + +#top_bar a { + -moz-transition: background 100ms linear 0s; + border-radius: 4px 4px 4px 4px; + color: #FFFFFF; + outline: 0 none; + padding: 4px 14px; + text-decoration: none; + text-shadow: 0 1px 0 #000000; +} + +.hidden { + display: none; +} + +table.tabs td.form input, table.tabs td.form select { + font-size: 90%; +} +input, textarea { + border: 25px solid #6F6F6F; +} +button, input[type="reset"], input[type="button"], input[type="submit"] { + -moz-appearance: button; + -moz-binding: none; + -moz-box-sizing: border-box; + -moz-user-select: none; + background-color: buttonface; + border: 2px outset buttonface; + color: buttontext; + cursor: default; + font: ; + line-height: normal; + padding: 0 6px; + text-align: center; + text-shadow: none; + white-space: pre; +} +input { + -moz-appearance: textfield; + -moz-binding: url("chrome://global/content/platformHTMLBindings.xml#inputFields"); + -moz-user-select: text; + background-color: -moz-field; + border: 2px inset threedface; + color: -moz-fieldtext; + cursor: text; + font: ; + letter-spacing: normal; + line-height: normal !important; + padding: 1px 0; + text-align: start; + text-indent: 0; + text-rendering: optimizelegibility; + text-shadow: none; + text-transform: none; + word-spacing: normal; +} + +/*logo*/ + +#header #logo { + float: left; +} + +#header #logo img { + -moz-transition: opacity 200ms ease-out 0s; + height: 78px; + margin-left: 10px; + opacity: 1; + width: 250px; + left: 10px; + margin-top: 5px; +} + +#header { + margin: 50px auto 30px; + width: 960px; + float: center; + font-size: 75%; +} + +#cgit { + margin: auto; + width: 96%; +} + + +#top_bar .left { + float: left; + margin: 10px; + padding: 0; +} + +#top_bar .right { + float: right; + margin: 10px; + padding: 0; +} diff --git a/roles/cgit/base/files/cgit.png b/roles/cgit/base/files/cgit.png new file mode 100644 index 0000000000000000000000000000000000000000..5154b9678fdcfc2d03626b9cbc3bf9b452f4e049 GIT binary patch literal 16124 zcmVWFU8GbZ8()Nlj2>E@cM*03ZNKL_t(|+U-zD9b^u5V@BKcy%y~sg3>6ysrHxp@4R%-Y~-w3V`eW!U$m_eo=&hjt!X$}zu6=u_NyXZ_8?EM>Tk@~lCVF&Qo0 ztaYA6D-Ugxdrilo?%t(9u>u@LV3(7DpKLD!urfHG_nR;WvLb(@3q9?W5&%16Gb!|$ zVi-#tvoSfn;%F;n<1wZkOlU{k=3vSgB=wIOw3R}ek>?@hwIcuH#wuwcDup%;7Jgix z}%Gij{7Q z34@Wg*_3iZE8~6IDV)V^W%yarqD6{?qfxXW>l9Z)Uun>PZ71!dzM4oVGHeerq$QpvuWd8C1)T03oO>n3k z`RqJ+)b@;F-nF{j0BF*^hezJu0etTc;LY2BDW+|(M#&V?Kp2Nh(MmhPd=AP&DM{2< z%qe0DE+vU#${{@|rGJcRSCVKx8jK?8S#jf&XOqd0zEYU2asbVvl{8J-D}%OVVN;{I zvZPn0s4Osm0c}hV5}DFvg=*cCk?btIq(oh5J@C~mIaIhumZqA9xWS~Qvz;viBt)!3OA}0EU41}+!p?>xFa~R%S6!8-E)kD zZ-6`)QWN2Jl@uU`Mpgg`;Aj>~ivWC-R#B>sS z2GriwvloCR5L!V%atD+cZ`-=OZNZAi`WDZA0%oQ}QvlqA0p(#BNuSvUOqGV?Y+^l9 z3T=~3ouwX`tdXp7QI^#8H2{}6Ke!#hOacF61Iq!d6@UV;vw%|xoZABI1K=aq12`S9 zOMp#aOJov$N8l3#z5t*@z_|eWBlL?E;P=rs7c)*)nsCWO&%UXtaqOgjlXwxSg2O)C zpOTP7_hpEdT)B)NY3o{XaNnZYPbQq?X&gQL*9zSe73a6tcOcq-3;F#uG>Xr{v_DRw z6@Vm?6rEueEQ?K&-3j1{l)1}`5hR&UqhoHv+$YC$^F^_-EL=btl0?=#rK!d+l!BAm zY)U)&+8bLX_4zUYHvVn?ZDor0I$R~v)aoqRdOOF$JM%HtVwB@&V#8Q~Cg z{|kinNSL@Jzks+&vLmmVaY`YF)g#K>;!fhE*KwC+2x^C)aiBY9Bms#a*A?@v4Oq3&s!X%!M`f$(tIZ1Z6V#!Cvj1h2mA59x!)`lOU+B* zh(K`pLoLT3AMYFDCaYsVuGCS+tdKze&7LTpK65e>bg6_)Qu3 zMg{oS0pMZpivb6uMGoelW))>IgzPQ5-()HV?(O-+r=kkV+l*^qM5B|RfiC#K6ioSTF0iHfd@G|$*Pw3YEp zWT?U>v}rI0Nm3IH#x6}ArU4d|bPq?-hdj=CX$PLC_CSDV?zH2uyb^#jfVU!@$`c@% zwlDm?6m#MRa;+ahn-&p|5kXG82;V)rH80Ej zTR(WRao9Px5a4_glM?1r1ZX>7Jxbs8(DBwT4 zAHeOvz7{aK0=#sZgPFhvSjYebM?(DRCjf2*b}jc#|M- z>B@z(zYYLau?co%C=2w=H;A>0(UeU%>SP7n1!e1(s^$aw8D;l znY2wYDCNL|Mk7_EnMJd_m9tc>H0&%7GeXEpk|h=I2--*h>Z7#7ZSe5L{TBj_-N118 z!8Zcq0qjL!oDH;i&$3e_8Ub$;Knt+@)>?e!063t&9i7`^j?D`Im^FXt7=p*RKOkHK zF5!4cQ39mjf&4z;c)doLNEWv?aR$r7eA4crAtQi+zRL{Y5g7nl>gJXYwz}gj*Qd{c zx1);BXoA_YymGTPw#kM`n!_w$GtWGmgiGuwRU|KMz}!k_a#-O};K9Rz^#IP^8F=+} z;4uM1uAQTmz_9=hjy|ZAG6r(PZ!F*kw*%i4FlYh00{Dc0cLcl{y{BjZ0M(w1Y&Z;C{eAHA=bFqz4Vfqajnl(SY%cTKuB zv84e1+(2ARJGqm*S)-nm%s5R=Om|7lE6F=H+6EJsX%iJG&QusS^Q8ubqDd?cFKhnJa;B{|@+dbO7!MoE6u3_jWl_%XI)S8L-xe#@Pb+YrtOSJgeq;y8?J4 zroZ9%6qEQ2h}T(<&y0@%;B!aNc; zFY6;xz71e=f>icK!rT>e=&$Y>sS@ajBk}-0U+WKc(Z>9lgCoo08^`CO0eVKykWjVkl?zLhLecg%jBjI_lg3LN8vpheE^kg}C2NW6l zt_(y-6H(U0OL8i)MLkv+}#t%||iE zf>UsN7-7wwKGM{)GO<0yrg5kYZ8lEPXO)^@mxoMgn@BpwQcz>-R19x&T_@!v$F zY(mJecBsH(Vy%gMPi8sa8sb^0Ap^}5XJBySYGqhW?6ecL)G|Tkf&MH>A-2zjhGtbq z2Gg)JJI86BHzQx&-w}Xc0tNt_cN1`7Bd|AsvF^jRRe(RA0c-@^G-&y%4H)l-I?q-X zN>sI+_)yuXcoAZZXCjp};uChCiT>CAU~+;d>aUd{M}uqc>S>3QyLMj(GeR>Q5NRWT zmjFD`yQuS5c6rEDa!K=A%)>F+*t!#jx2)ChmUl4goI6_zO)c$$4g&CQ0+Z!92MtJg zhs0APR;^sv`Lx$ftJp->|3)Ep5u}6)wJ3(;W{KsoeJ(V#pgJ<(KdOCx&tzEm5QGCk z>k6S;>flQd1%)y*kWTVD#K3BosO;*djmKG#5*I^ zq25QAoH!EK<<@EM#g;XH0szgLf6pXZ)=>oa6fh0cG}@RTSkJ(!UfZ{D?yCk&lmX+J zi1L7AKd6oj;@|~WaSUgCSmME;-b-+@O*Yfk0Emr*{w(kXTk2i4c+T430Bl8NXiM7W z&63~}f$tUoXFBWt@uWprO=lc=%=L9+6uUVN{q*&vhA~qHyOE41iN1OmTd( zq84g;ZIh&JB<}`r&9-%aefa66U#nI}wvEPZJ6p7mq>Jq6UePY-LI6iWZ1IN58#mXL z41UZb@mCx6e`{#yx>eVlRO6x8-qmvhiL>IS%+339d+(yoQ*&Ypb=R!(?{2jUg}I=0 zHlcP24Ge{G`)!wZn4l{01pqff_N~2(=e(7ZcI>G<>od0$t(I|@6PWL4a|EcDCHqqv z8hUuuwI|m+q;;3~^T6Z3^+D454^c+}uqeibp1%d+V=EVS{t-C}t{wWR>k19ycWMU; zX91WeX@X^iqOm>cTs7Gh1*%vdS~^WyZ)6&u7B}?%c`oD(@re8r#cQf^V(9= zxM_!pya2##f6^c~^s><$rc*(p$_?cvLiTknmO~cqJ9ND@~ z2cN#A*fMd~Hd=-MlytPW3FT}BUl(-7(;Z*A9;yLv|0`f$II?ZSvd3=t^sviFt|xhR zQtN~oMv~^66O>tG0a8@}&^G_>-Dnk00QhwhlLJPPJUnvIc7`!C>2leX>-&}*_gv^S zH}y^NKWpz=ehq1zBljo%I03KOcDbW($#HK4Om_K9fZrndaY67Hh!9F`A;Qh~NeMsi zT{L?(ts=cNp-Ejl;+$Js3yousBKe;n4z1DgBL``;H}V_?onhMSj7{~Hz9sWs4yl8t zn0lI&0+?r}-=0zyiN=tcY+3L9*w-upXA}64e`Dx0V!(76HGH@(+^jt%!w&q;wg&jU zNqZf(ld6?ufQOdGP5&njZ&{1>1^4dSIBxn+1^$xKY!7a97qPdD84e>58&m1 zCcBfVxTI7O$4A>!6;~X7?ww8LfsLS5Ov@@(*S1;n?>Pv-i8bGe z5+zZ)TUP1pkfioS(t3d8$0hj@1J08iYk~^Y3nTy{;46*50-J!^Ni7 zWu&!CcYp?TbNi{O#&i77IC5Y(UDu_>()MBji4291^i^S;$j-i-;n9++2* ztg)G_T-b^BuH}0b3dJW#oFt*yJ*!cRWQ5uk`qT@RahRec(&Njguu#0~(6eqRct~cL znK0sc3j1H#kF5EK*ZWaQHJ_&JGtlPx&5y{6&>`XRD3lGo>gcdAhdU__yE9%2<C>nKg61^BZ*L)5y+oY#L$kPt! z2wMIef2d*X)Qbc*5JGJLF#TsRTl5W_+w2P3yLu*)xC0IbQFMHR+=nuHvzuKv#8pB# zl%Ipw(<%I6CfX!C)V|=}_sFiuDmEZy!Nx35jv-)CE)#+GXzwt>d)m8tP80DhfQ|H6 zjpX}B(sNVrTJE<~0Qsvw+IPw~(AbPc8Dg<<6giBg>^A zglRi#p8h%JEm8Mb1rQ6yPTT9|_O70Bva4yX5=jB8am*$uC1e(`nnpW~vH9<<@B@Z6 zy=9a9xfjyw3xZypZzvZ=&??0;3sq|Iy~-HQvb(9htLG>HC^n8|hIr@yEjEsYT^&(- z*UzVu`YFj>U95w3V=V+ZYMnziK;%Lu82ZERo7U-~Dho&);*1~afKTs#0^_LDi`%<; zCIPscud4#JAFaYe?ep*1&#sK1P-@PU zNPIr#T9mzH72fy|8V~Y1f|f$cqVI6+Ggn_9XDA8zGqzp15VZlN^x9o$9P>?(my6yh zMHkDGpRC+fS^~)fp8A9!i=BUv$!EOYI(2jSNYE)HZ)@*bJ{I|k(Pvd!Oj-+RX*NtO zHHj)#4My_PFMlAiuuQ;}E;Lw|!Hr>g(UotwG zij5fBv_|b+_wFL#rz9sjr@JTpd(V_0e>yfvx=zx`l6{11`v6FXOZJJ9t|7S*-bot3 zHbH|V5E@VNslPbeoWfdDud@E2H-s(Qez5(EVt`MY<Vdk>*& z2rQNOu!KPc>GkEFUEWHZ2OF*}QxLm?%cC~y=-{N`Chv~LXd#@aIX8ETHeaT@$3Q903 z0td>m6gmu6k4wBu)K!4)7WJef9|eM@52?Di2_#R^k)(g6a6#L7%Pgzdm=p+_hS$*) zdqR4kYFaR6md}>&N8&HG-(T^uK>!QdyO!SsT1P<|j{>;s;7{Jr)iPn~S4sW5fR?cC0QfU30{dFDZUOKyyE2?+ zwyYWK_60rPrp06A9?z8jXy0b#=Z=&87GV8XwL0{ZRac(4J~|smeE#<4V#AnD0AB@g zuprBkr+w=f8S+*0@{`CI%Yh&p4J{K+6ZAR1>|w1bS{@Q6aG22jATM9JaP~uSMB2N1 z_J{C!kY`G0rEiJ~8TCA#WZ@8^bm>2T@fYX+{)ZncBPA-Aw%aD6E);{Fz7XI~E-_Hz z2w}e=agl9T@9bML_s#gR@vU_Nz%6V3r1k~N-%nYW32N~S3JlItd9)?k0A7G_MR{=3 z?T=o6#?~l-w)2;bXQA{-kli5n_L3KSil2Y%I)_d{(*|@QH@$9pb2#(+NKx}&B=~&xzdsoj_ z0bJ%A)?52( zY9?DJ_Ac(cE942Ac(r#e-v{7>Bxfk(+ZkvEzGJ}_lGoTH!x#5meZtlR^pgmUcY9aQ zc>pd1u`8nv@li$+2qH;-BtC!b=7T(^~^EnWUTo3RX z&{W}~ufBaB0MsYpzk3&VE=%!aiNyjA{mfD{jA>c{=vczH7W?)=Kj90xPPQ(WhJVnv zNs}^;RWH|AeZvHG2 z@%F_3)Jf}B9T2A~6x5KdYn+5)^qY_r3o^AZd6bFx-?cBecUKZ;xTk=Rs`*L4947LT zgp)ib0)T`4t7D@|Jtk6Y9LwHCozDvP@erQ#73Xlx9bfywR}VY)j+T%+eN40}v3tko zEP$UNIZZAQ6G5jO6Cp7~;QW;fXMd&d>Jzq(nX+fR?&>br1K?|Hai&=AK!4noW9%E&yQiO})7a+!#;MyeH$iO^~9 zoM(my)}I3K5_)#NT6ZEhe+>cP`v4GPBK4xU3z;$mB1_R75HJxD7YUf{F_9P*A&z0u zDmDrL8pkvoPvBTjKyzS`fPc83y(a18-o>5Y>05H*P@&ODk?ML``+e}3h*fMv-;(3D z_AcuDI;7Jft##d6T%RHNQoM_AAK>Oh1q&sbxE*AuB1!2U^P~%ffmzve#jN?zN@++) ziLEj6BAhMFrc4}Y!W&al_Gt$#YmR{T`R90mNqSk7s-RBmTip3)gAUeV8NG|zg4}9<~$;Bj-YMyGHi!~Bo(rX#hN*5U$-wdyE2Sf=iS|6+xU(H(Atj0Elk^)-F|?df9~G@{H@oT z>hQKyb8ARN=5zOc^FEO02^^3Df!JB#Zsu;;5czNI=Pqpxg{UwCe1!q#Vd{vc6l-5* zT$<-c;{A;)6Ijy_$vvdJ{H$S^oq$!(cj6FZg*WED$6nYV;4`j?PrgAT(6~Y3^4^7W zo@g95)ije;4c$`nc=j&td_~e%6;l_+-CoTkKYZv}H#H`heI15VunrLPKKTc)YUB%w z6t&cmEMg}#Gh15o#NF7taQ1T)|BqZ@JLHA!;|Fndv|`m(i|2wU`NlNg3gVR`*H%dv z_FZ-Sni}&$m(U%e&Vf-SDfq6M>0`h#! zoTm`HcH7#QuPe1oWF*H<(jimWtc%$I03K9HL_t)?ysct!u4P$!I~>HKYzOF-XEC+~ z0@wF0oc%!m-=0b;Ym$avsQ+(I*ekC-`z}ehy8Q~!Z3kF(3{bXqXG2qGigaN@J1IRt zo{9`{2WeWRJzoN`k(?;GcJM53{#BDkz&~@8$mNv4?I2sbQJm@6ViYnA`9EJP>1F^J z-uzA)xR^AMD7(?Zwq3rC!kc7m7v&l+KpfIAcH-DVLyO`Zxtzn%I*Oj(MZksvOpe$L z>C54JkX;t$lY1@^%Ks?S6lx8%6hAQ!^35>?E2eL;l zmav&o^IPDWV+m^C>&Pzv*`+>(^cvYr@PkrJ}sTU*H!|+;D*4kb^1_Tm@5QQ&MgBQUSH!me;BNQ$dW#u3j1}iBunjMDI#(d{!=;eKpCq6;7QJq}3CEv4C|{65~?@C$8CW zzCV+G*|&J^ODQTqX1a(m6ZhO1v}Pid=r61~x`6f%dY8=E5L)nQwS5z4J4|lXl_#tQ zbgf@$)ytP2(6%7Ff9if6lQI}X(k4%HqfhfiNvnmd41W~A97Ue`x~}ou3ONfz+iRIR zwuHbnoap&JdGRM9*B@KZOQ2Ob^z0iO8(XIe5Qev`BWaZHR45FY^WL|mKOx_Wj2xNo>XL|{b$Y#U_~ zzkTwS`DH_ApUk`}vh-Vs7l0ZY1f&^-Zffd&AEto!6Py@b6Q0$B;O|y0p8M*-r!B&h zw|1rZ&}3#FW1$FDD}Zz($*(EUoe-2fViC|`ZS(JG>RU2zD0=8v3X5E!usz!g{USu& zkkE#-WLf^kdC$LoMOLx27hvz>OV+B9`bcTSwku0h4#p%J$$4*xYb5+1$D=}^yI%s@ z+M9*a3;-JpoyYQ6bdoGLrWp$p_j56%SIUeiQIoSZGD;bFj_PhX=!_o~0lbe0MahbO zppP5e`w>WcHIAJ^*;Rl*fDt39`v4r-(#1nM&6L(>!_Z+z!fbqVO$#dxrxXIIlo7Sv0Fq3A{4)2iH1^t;8 z_i_D%5LAdF2(%Wg;-rXqJgd5&Nw$5H^p?ypn;0Gx}GZHx=Ini_1^wIZP*`{CB z^6FGIE9aQ?{pf^E?Oi>q0qqrz#x?-))jD+_OLlcn`R>JGf!k+7&t;r=OhENh((>8Q zIFfp#sX-XtltQVYB^GO`$=IaYUFi(@8^hIsat@0`-NOIG^?$+3zg{sdX%D${{`ln0 z=T#!<2283YCS}IQ%7wEB+q-(62e41w{5f}(B#j@p%l_K<+T$i0*(6mdb2^!sWcX*) zYLli{#$gSq*60{lZKkbw$(rbHYR2}4fvdeASK^}i!GEUz7ea9kh{mSWcEowPTM`2# zxl3Y1zXK5sRm$78V02fS7`rJcH%Qt}8s56j0x>QgfpAbSk{b*`Rv0o!(JI=K`ct%k z7TMK?I9CIk9zg5JGhQfO2+&^pRjn<+jfSxYIEwv~6KI~%(u62bI@Rs64rWyqHQtJyev&pScy}OB^$C-e)QT;j%0`n zStjf_U2$`bVcVA?a_R+66L2S$7RG!lwo z)Q*(z3Z*6i8Vrp@4PEyE6LQj^*&uO3RD6i^x$x)>XVQQ+4WInu=;*5L zuOA8>%CuuFs@KyZ*16I50Su_HJ`DlfM5q2}%#78mY>MdBemJ_XV@E+4KextON!LpH zY0|jZ(wmm!6h(xkqafg>sbS2-v2c=5XK`{fs}=y*7+=HoLm@blfP`dbrRMQAsQy5F ztn)Q5fSi

YJ!uHAy5IS|)?w_!@NA$EFBS+oVc44r5`BH>|hI6nNg(7ferc!iv>l zx4!){L2is5Ho=q%a7sKNA}*)l25IFiLQu1&d)6mO*C1Ggu+d#Hws|;zU;X5YG zFtgTAF~iE$39@N{ux0Rsn9ts`!+0(i)Ik1&(DrJD39Sy#StyWP2i#b48>|BUVGx{uHMQfm42T zBxCcD$j0lB-KpJYed;+sZoQ_ws^{LyV*p%MXdJ8R@Kzvyjb;C8ne;9UZ1@`>obKl3 zV8Uv6O?G8-?6VkBfIKx5>Wk6R z_aHorz+mx)xLiCvgUkj8jRBK~Qm=sgMMUpK~xqe@2|o5%*h zt==344k~Ug52+w==HZ{avt=~;s4-UKAEwnbz>zDRIwgN}7BnKZNv8}7z>lo z0zRd1W~|z<`3itXAHDul!*T16F^6*~J6={DE}FD^aN`@S+SOM8{3WUM9e^UGGXZ!^ zgobk(#UAyyM{aQsVx1c>ddB;#OC~BuQvWQ&^^X()J-3C?N|}J<@ePY(NUIH(Y61?K z!*s99H5iT<4duT)qR`N|XNGv0^fwMj5&$)h-HGoy>{K3q<+eKgBm)7z*9|_a8y7d& z>vWfvnGQe}UWD|QXr!>~x-LpZg8vyoZB61Z zVl4_2cR#$F(4>UYdnCD4updl1w1Rxy^ce^cBC^7$UNOV17v!|T3eqro-nD@~$E3AkMcNYWmyP7Y z@M-u6Q}$IH!?BVWi4o$7;w4r4xk6+rBV!8Qk;}_oMu^t&xG`k zIMFW6iSEfy^|t`VBTLc@gz9{=@lJ8eUi!+(G9r>`qYx`a}*mxhiZoKaDQCRrV} zk)0wSE!nd&0BWXSB1&2hRsylahK<(OJMp=+u;uL+%K&Z!WCt}Iyo`GE96%Pdb+4EU z0IfUklO;VQ&v3PjBhJ02xoPqq|3z{+$ZvtXTGIU;-90CHU?Eb=Ian`>db~~ob@!yZ z>Q8O!>bU?wG}~HcJBx18oVKo>e-?15pR*#L3RyGeB$j}vFp`T!fdfsmgv(}iE#DOY zn#N5tQB9rpHnl?4GWMDGbqTShJ;GGvxY0cevB;ynH;HFa%_7fjd^R=0~4%CBi|{sm;?nPvZu zYGv>~0*}Sjv2}BzmI&S0-gWPhTh_b;q`R@yJ*IKo&IEv2=Phk1G!%bE;0pj+LVhvt z0{G2Y=PlhiL8SL1rB_~Y>?R1efGF34gdtW0FviG||Y>g8Ke$kJWoZ;_2FB}JTv)e-_=-@@7bBrft6 zw;$_OtBbV)`f+<#&(Uu@u@sQ#GQlLZ!BJK03fDY)Kia!`PN4835=}u_3?M=UigI}~ z3@AxuNmK8#d`GhCaM%4Urf{ivQ;}atcaATXn&$xc zp#@A5ya?SCB&?ppM8?EZUSo^z@r0}ns^qmzeY z`X87zPWZ0XI_)#`c43d(D_ zNIG$y)DqhU?`R&|+y!ZtzoA3sb$5QpOL(BYtLMvhdE};5SD(<2G`DdHZS(J$WEBeM zO8B;F5`3!8<2tHwM5MLMd^;&-VIeDiF5b+m6w6KB-y+hZJL^vUXy|>NAzQiZ;5F~7 zBZD^;8k&!ha3W(#%Qdg!_G0sd+uOUA-v-$a^)8+_P!D4`)|ZY2%l8#I3&1BJe9X0h zD*AY*wvvWH>m3uhu5Zb_x7xdUejwm#C;Ufskbs{dQM?GiDIHxsHwoyIU3p=6^SX^s z{Pdji5vMM;Fmc9&f@SSVt8lo0<0u^+s+`QY#K-y69Kk9zqi^BtEgjt}z5wYuxB;6X z2Ibc#l=N;AzbZD4xvQgV#eJY2tyZgVu39p0izC(jtl26y?F3*Z$)fjoi zi*LzwcIMUP;Vn-zjG4@lZR=A-XtAcyNlhjYCbQElhEnI?)jDloJaNNGc6(RP1tjhf zG?A(;Ti~%15*EN8TBXKQ+ZOaJg~eW$D=+o0|5N{ScYdup^D{RW8yj0DT7}}i65B0{ zCqp{OO-CP8SE&h_453FMnQY9I-L>vzW69nmsdJW^l2^5<8Si2SC#z#7{tk@#W992) z9b$a=XK&G}t3Od`JAc{dSZJ6AsDqL18I<`|3D74&Yc9ao+q-(61n@_=K~|2MqTK=h zlYqTQ?gXI0Et*uGv;ok^lQa0LqGLJVk^1e`+PcmZoItI*%x)idBqCM_gdAx@Rl5*jUfL6!_x zt8##chXMP-N3J=ooFYp(Jj1p_RvRB$@%`8dBMYHYB;vqwv*tz6Dq#I953iiO&k^6G z_TrOELE3;~VcmqC|7oDDd&Ll?CIDkdP603pp;}CSisw+wwRbpp zrItO3Q5j6|W4@`4GG_O#Jf6n{mGD$&? zsEY282`W*Hvn#_Gv(w(#w)PbO_5~7W0_<>|rb6}h7r+t63kb|OIrbP4-%SanQ&RMx|?xyC6yJB$T8=2U>Q|!q0fmg})H=4{&SY(Ls zA;wk3i}zh>H{-x7KG6N`BP5>U@;^7ByaFKI_i7PEP9`wf9k3vWPNE)c=;Xa`7D7#g?*>vgAE&${@#v+%<>+ed)pb zbeZhRhx@MV9Bvr96Om#{_1FSwcX<4CwHp?mhsFMk@sga|;@ z>hz`w)4x<58JIm~pSI(+zWws#YIWpM15 z!*y(G+T{%7T0@jM4pH|L$n-AiTq$4o=u{nMqkbAPP#i2_!E7`N_T| z$89S#j={**wZ?&Jp|-wAVlKdYLj_!9Zk&z;89IWASN7x{+0_$4JzkgkzV=#gav)r` zFEM7Py|HrP?9H?ar-8gUj1Y!_^O(8K7M@X)8TKxKAMah%`KOOw{4`$v)#XN&+X!qT zScMd`^cC%E)+jT@2O@Nc4Wo71zUceWoRQ&8Yd;U+YXGVa3>{5Z!rVrCoBxa_M(56KGObw9YZ6cj4^64sTt5hNKHX4tp{FE{hL->U;BqM9K(D z^iV`s1R#l3va5Uk;npvFZ{-!o4j`yiV#1DVzh zfVcJN9mw8IxN&&%x|92s%-u{|KUBDeq*oo&b9{hH|3!OEQl#V=2BZSJEo1ItOFK5U zPQ~z+cf4B0UnlVy05e3E0r~^f6PvMS=-jWYS#)W zsp^K`&b9?BuCSQ`c2rnqIYrU_ro+g7WAAi@&|x|vVabV@*BG{a#G)w$FDzP!CjMy0tZ+B zbwkT8drh-7vb8+0dFY87&mPHu3$oH>A_Fg>;WIEQMp9}XkMiIq%sPMBG^^Nf5rN|* z?M(7*Nz3hO?Wdh0r{U#lN$>yD~`%Lt|_6KJ(b@j{?^e+HAK(qmv9+r%J z8-#r0+FPKWg7C0lKe+zsCx7>+hb~UEZWu6<)J4fciC(3&j;@{$3up(?POuHY*zg8a zs6wT{w;=t71q74JTxv7f^5S zqB&1`?Ur3J@s_Un4GSeX-p98!7fNIQ4Zx>C?g?NMq=#hdqP`__o@y8~86(@)o0>pP zHeKC);d;ky=P!G=RcQF2fDRBxLE5c0SE5329ydzp6ZC7yzPoSn+}8jgye~d*#-d{5 z#67-1@>~M@2-pPRe#pM0chQ{ZqxGQkuEP{74AK~QlEjhWi$dm+L;%2JCLcKSx`w9K zsgr53r82Z_uu>je|M-v38s6ogd3f`wdrY306=_`4=#T@GcRDv$E5j3M6{nNdejukq z8VjI8U^BqiWmo@1tFWOmvTf^2zr1S8hF2d9Z*~*fS*LpJZ{1Dge|E&VceWOajT2y5 zyF=KY^ zC;9DeowhGrHNoVxtQ%8rt~=WP5YGzhx?zn(*wj zN?~~_E;EYm!@X=w!F@W8D5jRzzve*rTF(`3ORr8D?D z(Kb~FOiga4ec~t>P#PoIp7G4n?>q+(a`e~qATi0$VHs@J9Zj1Fh78j=!w#UGb$FQ0 znd!lYLJG&qszx4{{~@auMw-rs6bn*PO;^Dz*^H5QZN?ak_M}zHX2~j}AurFp+Y02=?%g>mS!YuKzz*R_WB}m1G%Z%I~u7HKqNJ>mS!Y zuK%S>B(J236e-=r&|re4)w#(3" .. buffer) + return 0 +end + +function filter_write(str) + buffer = buffer .. str +end diff --git a/roles/cgit/base/files/favicon.ico b/roles/cgit/base/files/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..d1ae2d209a4b11cd2b4d0cb21408df3a8498f0f1 GIT binary patch literal 3638 zcmeH}e@v8h9LHZ@j$i(YQ~Zg@4^K};9yxtXMKexCATR~b-v^~63MhD@f=rYX6n_;q zUAU-m%M8;}SZxM5)tK5=QxjHe+4=*Q%WRp~p69va4s@-x{^Y%V{&>Fc_vgp+ z`F`&690>f->rsf7xN_&1k7^(wcJfKLFV?nS=nuVt6375;ol~TC;!D|#wg7|tEHP{SX; z{YrDm08W<7;L{U!w#UP(3*l7d3R()Fel<9@CUP`Krm-T0y7ke#V>8j#eSI^VaGh!EW`PYaB}xj9{=@-QZu}s0q^ceW!p?RkOs~Z zmDCnY4J#V!az4#9pupxTP;E4;Kv*c;c`EslfB7WTZg5*r)Byq{mXJ@C1`RP{c zOID!B@5L!=0z}FyR;x8xeV#tcf3g_AaP9Nfg2fOvW3_RPPE0V3w&koZfV`=3!9u~z z@R-HR!IHBvbYzAYv~*b7%Rv@cY*{zYUoQq`C#{$iq9iD^TGL+;hJ;1Ao6XArOO|!j z*k|VUkL+ile3nmC;U zHydqSccjyL+)jO|nVL*Jw?1#6saVhN_dEIGsF}{Y?Nn#L`5GHn&mN}zqMaXZv{IF! zr#dr=CL7%Uv5n5#t<+^gRT|V}NAX#io|iKaJgv|nrGhy~8(YI|(we9v3;q8GjRA0T(b69N6Z(Xu?XheDx^ z{fC%zgGYFjz7pjudZ4;_ML&s}5NXm87U}F-(p}GM28v-1|fvbV3;&ci>6f|%f>KClKjTFTO@=Cmqt(4B%t(fT$m*F_0mEp z4N(OeFB@aReEmmC9szi+H#AbqZ!#DR1C)x?o_Ki`UG+!1d#a3rMs?@qDUu|)Z~3YG zD3w8-2W0_!OzYo4X4>il?wD zRM#$te+8Gzu)lxs6GGy literal 0 HcmV?d00001 diff --git a/roles/cgit/base/files/fedora-layout.css b/roles/cgit/base/files/fedora-layout.css new file mode 100644 index 0000000000..cac7604211 --- /dev/null +++ b/roles/cgit/base/files/fedora-layout.css @@ -0,0 +1,300 @@ +/* Basic tags */ + +#body { + margin: 90px 230px 0px 10px; + padding: 0px; +} + +a img { + border: 0px; +} + +/* Anchors */ +a { + color: #0000ff; +} + +a:visited { + color: #551a8b; +} + +a:active { + color: #ff0000; +} + +/* Basic classes */ + +.none { /* to add paragraph spacing to various elements for ttys */ + margin: 0px; + padding: 0px; +} + +.invisible { /* stuff that should appear when this css isn't used */ + margin: 0px; + border: 0px; + padding: 0px; + height: 0px; + visibility: hidden; +} + +/* +.left { + margin: 10px; + padding: 0px; + float: left; +} + +.right { + margin: 10px; + padding: 0px; + float: right; +} + +.center { + text-align: center; +} +*/ + +/* Common page elements: Header, footer, etc. */ + +#logo { + border: 0 none; + height: 100px; + left: 0; + margin-top: 39px; + position: relative; + width: 64px; + z-index: 10; +} + +#logo a img { + width: 274px; + height: 72px; + position: relative; +} + +#hdr { + position: absolute; + z-index: 5; + top: 0px; + left: 0px; + right: 0px; + width: 100%; + height: 48px; + text-align: right; + background-color: #e3ffc3; + border-bottom: 1px solid #807d74; +} + +#banner { + position: absolute; + z-index: 10; + top: 0px; + right: 0px; + border: 0px; + width: 300px; + height: 48px; +} + +#banner a img { + width: 300px; + height: 48px; +} + +#hdrNav { + position: absolute; + top: 54px; + left: 0px; + + margin-left: 84px; + text-align: left; + vertical-align: middle; + font-size: small; +} + +#hdrNav a { + color: #000000; +} + +#hdrTitle { + height: 48px; + padding: 10px 10px 0px 0px; + font-weight: bold; +} + + +/* Search thingy */ + +#search { + font-size: small; + margin-bottom: 10px; + padding: 10px; + background-color: #dddddd; +} + +#search input { + border: 1px solid #666666; + background-color: #ffffff; + + vertical-align: middle; +} + +/* Sidebar */ + +#sidebar { + position: absolute; + top: 90px; + right: 0px; + width: 210px; + + /*margin-right: 10px;*/ + padding-right: 10px; + padding-bottom: 0px; + border-left: 1px dashed #dddddd; + background-color: #ffffff; +} + +#sidebar p { + margin-top: 0px; + padding-left: 10px; + padding-right: 10px; +} + +#sidebar p.section { + text-align: center; + font-weight: bold; + padding-top: 3px; + padding-bottom: 3px; + color: #999999; + background-color: #eeeeee; +} + +#sidebar ul { + margin: 0em; + margin-bottom: 15px; + padding-left: 10px; + padding-right: 10px; + list-style-type: none; +} + +#sidebar ul ul { + padding-left: 2em; + padding-right: 0em; + list-style-type: square; +} + +/* Copyright footer */ + +#copyright { + text-align: center; + font-size: small; + clear: both; + + margin-top: 10px; + padding: 5px 0px 5px 0px; + color: #aaaaaa; +} + +#copyright a { + color: #aaaaff; +} + +#copyright a:visited { + color: #ffaaaa; +} + + +/* News Sections */ + +p.newsitem { + clear: left; + margin-bottom: 20px; +} + +p.newsitem img.newsicon { + float: left; + margin: 0px 10px 10px 10px; + border: 0px; +} + +/* Generic Classes */ + +div.code { + background-color: #e0e0e0; + color: #000000; + white-space: pre; + font-family: monospace; +} + +#page { + margin: 0px; + padding: 0px; +} + +div.in-column { + margin: 0 0 2em 1em; + float: right; + max-width: 12em; +} + +hr { + color: #888; + background: #888; + border: 0; + height: 1px; + width: 90%; + text-align: center; + clear: both; +} + +div.body { + clear: both; +} + +div.sidebar { + position: absolute; + text-align: left; + right: 0px; + top: 60px; + width: 27ex; + padding-left: 1ex; + border-left: 1ex solid #eee; + margin-top: 4em; +} + +div.sidebar h2 { + margin-top: 0; + padding: 5px 2ex 5px 2ex; + background: url(cgit-data/images/t.png) top left repeat-y; + font-size: 100%; +} + +ul.toc { + padding: 0; + padding-left: 20px; + margin-left: 0; + margin-right: 10px; + list-style: none; +} + +ul.toc li { + list-style: circle; +} + +ul.toc li a { + text-decoration: none; + color: black; +} + +ul.toc li a:hover { + text-decoration: underline; +} + +/* page content */ + +div#content { + clear: both; + padding: 1em; + margin: 1em; +} + diff --git a/roles/cgit/base/files/fedora-style.css b/roles/cgit/base/files/fedora-style.css new file mode 100644 index 0000000000..7a021f8ca8 --- /dev/null +++ b/roles/cgit/base/files/fedora-style.css @@ -0,0 +1,225 @@ +/** + * Styles that are not a part of page layout. + * + * For example: + * Fonts + * Sizes + * Decoration + * Separators + */ +body { + font-family: verdana, arial, sans-serif; +} + +/*div#content { + max-width: 50em; +}*/ + +div#global_domain_bar a { + color: #2E3436; + font-family: Cantarell,'Droid Sans','DejaVu Sans',Arial,sans-serif; + font-size: 14px; + line-height: 1.6; +} + +div#content a { + color: #3465a4; + border-bottom: 1px dotted #888; + text-decoration: none; +} + +div#content a:hover { + border-bottom: 1px solid #888; +} + +body { + font-family: Cantarell, 'Droid Sans', 'DejaVu Sans', Arial, sans-serif; + font-size: 14px; + line-height: 1.6; + color: #2e3436; +} + +body.win { + font-family: Verdana, Arial, sans-serif; + font-size: 14px; + line-height: 1.6; + color: #2e3436; +} + +body.win h1, body.win h2, body.win h3, body.win h4, body.win h5, body.win h6 { + font-family: "Trebuchet MS"; +} + +h1, h2, h3, h4, h5, h6, dt { + color: #0489B7; +} +h1 { + font-size: 42px; +} +h2 { + font-size: 25px; +} +h3, dt { + font-size: 21px; + margin: 0.4em 0 0.5em; + color: #2E3436; +} +dt { + color: #E36615; +} +dt a:hover { + color: #fa7721; +} +h4 { + font-size: 16px; + color: #ce5c00; +} +h5 { + font-size: 14px; +} +h6 { + font-size: 12px; +} +ol { + list-style:decimal; +} +ul { + list-style:square; +} +li { + margin-left:30px; +} +dl,dt,h1,h2,h3,h4,h5,h6,pre,table,address,fieldset { + margin: 0.8em 0 0.4em; +} +p, dd, .action_box, ul, ol { + margin: 0.4em 0 1em; +} +/* comment this out for now. See: https://fedorahosted.org/fedora-infrastructure/ticket/4235 +code { + background: #ececec; + background: rgba(0,0,0,0.1); + padding: 1px; +} +*/ + + +/* Text classes */ +/* ========================================================================== */ + +.highlight { + background: yellow; +} + +.main_feature { + font-size: 16pt; + line-height: 130%; +} + +.footnotes { + font-size: 11px; + color: #888a85; +} +.footnotes a { + color: #888a85; +} + +#footer { + font-size: 11px; + line-height: 1.5em; +} + +/* lists */ + +div#page .list { + margin-top:.5em; +} +div#page .list tr td { + padding:.2em; + text-align:left; +} +div#page .list td label { + border-bottom:1px dashed #999; + font-weight:normal; +} +div#page .list th { + background: #ccf; + border: 1px solid #000; + font-weight: bold; + padding: 2px; +} +div#page .list th a { + display: block; + padding:.2em 1.2em .2em .2em; + text-align: left; +} +div#page .list th a:hover { + background-color: #fff; +} +.row1 { + background-color: #eee; +} +.row2 { + background-color: #ddd; +} +.row1:hover, .row2:hover { + background-color: #fff; +} + +.record th { + text-align: right; +} + +/* styling page content */ + +h1 { + font-size: 1.5em; + color: #3f3f3f; +} + +/* styling form widgets like bugzilla.gnome.org */ +input,textarea { + border: 1px solid #6f6f6f; +/* background: #dddddd; */ +} + +input.login_small { + border-style: none; +} + +input:focus,textarea:focus { + background-color: #f7f2d0; + color: #000000; +} + +/* select { + border: groove +} */ + +option { + border: 0px none #ffffff; +} + +input[type=radio] { + margin-left: 1em; +} + +/* Syntax highlighting */ + +table.blob .num { color:#2928ff; } +table.blob .esc { color:#ff00ff; } +table.blob .str { color:#ff0000; } +table.blob .dstr { color:#818100; } +table.blob .slc { color:#838183; font-style:italic; } +table.blob .com { color:#838183; font-style:italic; } +table.blob .dir { color:#008200; } +table.blob .sym { color:#000000; } +table.blob .kwa { color:#000000; font-weight:bold; } +table.blob .kwb { color:#830000; } +table.blob .kwc { color:#000000; font-weight:bold; } +table.blob .kwd { color:#010181; } + +table.list td a.ls-dir { + color: #0000FF; + font-weight: bold; +} diff --git a/roles/cgit/base/files/html-bg.png b/roles/cgit/base/files/html-bg.png new file mode 100644 index 0000000000000000000000000000000000000000..b51984ee0b59d14bab793d17003d75b302322e4f GIT binary patch literal 312 zcmV-80muG{P)Cf_Z+I!T2jSOmT#BV$uO`j`v zHZtwG-IOQ)30E$h4wR%9&hv_^@BUT_{Ufh-pYe9F+=jX%jg1c_YX&8~DJA{+{TToN03CEi zSad^gZEa<4bO1wgWnpw>WFU8GbZ8({Xk{QrNlj4iWF>9@005y$L_t(o!|jwo7K9)S zL! zvV2eaYyPOuv}U<3o;~I?>(0x9Cac@|G^_b&HQxBj)_gth+vlF;iuJtjfgb=}4Hh)z ze>Lzm{U&ejRdQ!X`v3Kxypd#28n3+Fixol$5l~|gLZ~qaA=DUz5Fa=IVu{r6`peNK P00000NkvXXu0mjfPIrCC literal 0 HcmV?d00001 diff --git a/roles/cgit/base/tasks/main.yml b/roles/cgit/base/tasks/main.yml new file mode 100644 index 0000000000..abb35894eb --- /dev/null +++ b/roles/cgit/base/tasks/main.yml @@ -0,0 +1,26 @@ +--- +# tasklist for setting up CGit + +- name: install the needed packages + yum: pkg={{item}} state=present + with_items: + - cgit + - lua-md5 + +# See https://github.com/robyduck/cgit-custom +- name: add our custom theme + file: dest=/usr/share/cgit state=directory owner=root group=root mode=0755 + file: dest=/usr/share/cgit/images state=directory owner=root group=root mode=0755 + copy: src={{item}} dest=/usr/share/cgit/{{item}} owner=root group=root mode=0644 + with_items: + - cgit.css + - cgit.png + - cgit-fedora.css + - fedora-layout.css + - fedora-style.css + - images/favicon.ico + - images/html-bg.png + - images/t.png + +- name: install the libravatar filter + copy: src=email-libravatar-korg.lua dest=/usr/libexec/cgit/filters/email-libravatar-korg.lua owner=root group=root mode=0755 From 7e979e4050f2a8290858f9f3f68cdd92d72ddfed Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 8 Jul 2014 15:32:17 +0200 Subject: [PATCH 119/193] Add a new cgit/clean_lock_cron role This will be needed to migrate Dist Git from puppet to ansible. --- roles/cgit/clean_lock_cron/files/clean-lock.cron | 2 ++ roles/cgit/clean_lock_cron/tasks/main.yml | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 roles/cgit/clean_lock_cron/files/clean-lock.cron create mode 100644 roles/cgit/clean_lock_cron/tasks/main.yml diff --git a/roles/cgit/clean_lock_cron/files/clean-lock.cron b/roles/cgit/clean_lock_cron/files/clean-lock.cron new file mode 100644 index 0000000000..050218cee7 --- /dev/null +++ b/roles/cgit/clean_lock_cron/files/clean-lock.cron @@ -0,0 +1,2 @@ +*/15 * * * * root find /var/cache/cgit/ -cmin +60 -name '*.lock' -type f -delete -maxdepth 1 + diff --git a/roles/cgit/clean_lock_cron/tasks/main.yml b/roles/cgit/clean_lock_cron/tasks/main.yml new file mode 100644 index 0000000000..be680c416d --- /dev/null +++ b/roles/cgit/clean_lock_cron/tasks/main.yml @@ -0,0 +1,5 @@ +--- +# tasklist for setting up the Cron job cleaning CGit locks + +- name: install the cron file + copy: src=clean-lock.cron dest=/etc/cron.d/cgit-clean-lock.cron mode=0644 From de1a1701c535ece5cb2c2d47826a5ebc5e41d560 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Fri, 11 Jul 2014 12:29:12 +0200 Subject: [PATCH 120/193] Add a new clamav role This will be needed to migrate Dist Git from puppet to ansible. --- roles/clamav/files/freshclam-cron | 17 +++ roles/clamav/files/freshclam.conf | 197 ++++++++++++++++++++++++++ roles/clamav/tasks/main.yml | 42 ++++++ roles/clamav/templates/clamscan.sh.j2 | 36 +++++ 4 files changed, 292 insertions(+) create mode 100644 roles/clamav/files/freshclam-cron create mode 100644 roles/clamav/files/freshclam.conf create mode 100644 roles/clamav/tasks/main.yml create mode 100644 roles/clamav/templates/clamscan.sh.j2 diff --git a/roles/clamav/files/freshclam-cron b/roles/clamav/files/freshclam-cron new file mode 100644 index 0000000000..61a59170b1 --- /dev/null +++ b/roles/clamav/files/freshclam-cron @@ -0,0 +1,17 @@ +#!/bin/sh + +### A simple update script for the clamav virus database. +### This could as well be replaced by a SysV script. + +### fix log file if needed +LOG_FILE="/var/log/clamav/freshclam.log" +if [ ! -f "$LOG_FILE" ]; then + touch "$LOG_FILE" + chmod 644 "$LOG_FILE" + chown clam.clam "$LOG_FILE" +fi + +/usr/bin/freshclam \ + --quiet \ + --datadir="/var/lib/clamav" \ + --log="$LOG_FILE" diff --git a/roles/clamav/files/freshclam.conf b/roles/clamav/files/freshclam.conf new file mode 100644 index 0000000000..e161995eec --- /dev/null +++ b/roles/clamav/files/freshclam.conf @@ -0,0 +1,197 @@ +## +## Example config file for freshclam +## Please read the freshclam.conf(5) manual before editing this file. +## + + +# Comment or remove the line below. + +# Path to the database directory. +# WARNING: It must match clamd.conf's directive! +# Default: hardcoded (depends on installation options) +#DatabaseDirectory /var/lib/clamav + +# Path to the log file (make sure it has proper permissions) +# Default: disabled +# UpdateLogFile /var/log/freshclam.log + +# Maximum size of the log file. +# Value of 0 disables the limit. +# You may use 'M' or 'm' for megabytes (1M = 1m = 1048576 bytes) +# and 'K' or 'k' for kilobytes (1K = 1k = 1024 bytes). +# in bytes just don't use modifiers. +# Default: 1M +#LogFileMaxSize 2M + +# Log time with each message. +# Default: no +#LogTime yes + +# Enable verbose logging. +# Default: no +#LogVerbose yes + +# Use system logger (can work together with UpdateLogFile). +# Default: no +LogSyslog yes + +# Specify the type of syslog messages - please refer to 'man syslog' +# for facility names. +# Default: LOG_LOCAL6 +#LogFacility LOG_MAIL + +# This option allows you to save the process identifier of the daemon +# Default: disabled +#PidFile /var/run/freshclam.pid + +# By default when started freshclam drops privileges and switches to the +# "clamav" user. This directive allows you to change the database owner. +# Default: clamav (may depend on installation options) +#DatabaseOwner clamav + +# Initialize supplementary group access (freshclam must be started by root). +# Default: no +#AllowSupplementaryGroups yes + +# Use DNS to verify virus database version. Freshclam uses DNS TXT records +# to verify database and software versions. With this directive you can change +# the database verification domain. +# WARNING: Do not touch it unless you're configuring freshclam to use your +# own database verification domain. +# Default: current.cvd.clamav.net +#DNSDatabaseInfo current.cvd.clamav.net + +# Uncomment the following line and replace XY with your country +# code. See http://www.iana.org/cctld/cctld-whois.htm for the full list. +#DatabaseMirror db.XY.clamav.net + +# database.clamav.net is a round-robin record which points to our most +# reliable mirrors. It's used as a fall back in case db.XY.clamav.net is +# not working. DO NOT TOUCH the following line unless you know what you +# are doing. +DatabaseMirror database.clamav.net + +# How many attempts to make before giving up. +# Default: 3 (per mirror) +#MaxAttempts 5 + +# With this option you can control scripted updates. It's highly recommended +# to keep it enabled. +# Default: yes +#ScriptedUpdates yes + +# By default freshclam will keep the local databases (.cld) uncompressed to +# make their handling faster. With this option you can enable the compression; +# the change will take effect with the next database update. +# Default: no +#CompressLocalDatabase no + +# Number of database checks per day. +# Default: 12 (every two hours) +#Checks 24 + +# Proxy settings +# Default: disabled +#HTTPProxyServer myproxy.com +#HTTPProxyPort 1234 +#HTTPProxyUsername myusername +#HTTPProxyPassword mypass + +# If your servers are behind a firewall/proxy which applies User-Agent +# filtering you can use this option to force the use of a different +# User-Agent header. +# Default: clamav/version_number +#HTTPUserAgent SomeUserAgentIdString + +# Use aaa.bbb.ccc.ddd as client address for downloading databases. Useful for +# multi-homed systems. +# Default: Use OS'es default outgoing IP address. +#LocalIPAddress aaa.bbb.ccc.ddd + +# Send the RELOAD command to clamd. +# Default: no +#NotifyClamd /path/to/clamd.conf + +# Run command after successful database update. +# Default: disabled +#OnUpdateExecute command + +# Run command when database update process fails. +# Default: disabled +#OnErrorExecute command + +# Run command when freshclam reports outdated version. +# In the command string %v will be replaced by the new version number. +# Default: disabled +#OnOutdatedExecute command + +# Don't fork into background. +# Default: no +#Foreground yes + +# Enable debug messages in libclamav. +# Default: no +#Debug yes + +# Timeout in seconds when connecting to database server. +# Default: 30 +#ConnectTimeout 60 + +# Timeout in seconds when reading from database server. +# Default: 30 +#ReceiveTimeout 60 + +# When enabled freshclam will submit statistics to the ClamAV Project about +# the latest virus detections in your environment. The ClamAV maintainers +# will then use this data to determine what types of malware are the most +# detected in the field and in what geographic area they are. +# This feature requires LogTime and LogFile to be enabled in clamd.conf. +# Default: no +#SubmitDetectionStats /path/to/clamd.conf + +# Country of origin of malware/detection statistics (for statistical +# purposes only). The statistics collector at ClamAV.net will look up +# your IP address to determine the geographical origin of the malware +# reported by your installation. If this installation is mainly used to +# scan data which comes from a different location, please enable this +# option and enter a two-letter code (see http://www.iana.org/domains/root/db/) +# of the country of origin. +# Default: disabled +#DetectionStatsCountry country-code + +# This option enables support for our "Personal Statistics" service. +# When this option is enabled, the information on malware detected by +# your clamd installation is made available to you through our website. +# To get your HostID, log on http://www.stats.clamav.net and add a new +# host to your host list. Once you have the HostID, uncomment this option +# and paste the HostID here. As soon as your freshclam starts submitting +# information to our stats collecting service, you will be able to view +# the statistics of this clamd installation by logging into +# http://www.stats.clamav.net with the same credentials you used to +# generate the HostID. For more information refer to: +# http://www.clamav.net/support/faq/faq-cctts/ +# This feature requires SubmitDetectionStats to be enabled. +# Default: disabled +#DetectionStatsHostID unique-id + + + +# This option enables support for Google Safe Browsing. When activated for +# the first time, freshclam will download a new database file (safebrowsing.cvd) +# which will be automatically loaded by clamd and clamscan during the next +# reload, provided that the heuristic phishing detection is turned on. This +# database includes information about websites that may be phishing sites or +# possible sources of malware. When using this option, it's mandatory to run +# freshclam at least every 30 minutes. +# Freshclam uses the ClamAV's mirror infrastructure to distribute the +# database and its updates but all the contents are provided under Google's +# terms of use. See http://code.google.com/support/bin/answer.py?answer=70015 +# and http://safebrowsing.clamav.net for more information. +# Default: disabled +#SafeBrowsing yes +# + +# This option enables downloading of bytecode.cvd, which includes additional +# detection mechanisms and improvements to the ClamAV engine. +# Default: enabled +#Bytecode yes diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml new file mode 100644 index 0000000000..335a2f76d2 --- /dev/null +++ b/roles/clamav/tasks/main.yml @@ -0,0 +1,42 @@ +--- +# tasklist for setting up a ClamAV scanner +# +# The following variables control the scan: +# - clamscan_mailto +# - clamscan_paths +# - clamscan_excludes (optional) +# +# The following variables control scheduling of the cron job: +# - clamscan_minute (optional) +# - clamscan_hour (optional) +# - clamscan_day (optional) +# - clamscan_month (optional) +# - clamscan_weekday (optional) + +- name: install the needed packages + yum: pkg={{item}} state=present + with_items: + - clamav + - clamav-db + - cronie-anacron + +- name: setup the freshclam configuration + copy: src=freshclam.conf dest=/etc/freshclam.conf + +- name: setup the freshclam cron job + copy: src=freshclam-cron dest=/etc/cron.daily/freshclam + +- name: setup the periodic clam scan + template: > + src=clamscan.sh.j2 + dest=/usr/local/bin/clamscan.sh + mode=0755 + cron: > + name="clamscan" + minute={{ clamscan_minute | default(35) }} + hour={{ clamscan_hour | default(4) }} + day={{ clamscan_day | default(*) }} + month={{ clamscan_month | default(*) }} + weekday={{ clamscan_weekday | default(5) }} + job="/usr/local/bin/clamscan.sh" + cron_file="ansible-clamscan" diff --git a/roles/clamav/templates/clamscan.sh.j2 b/roles/clamav/templates/clamscan.sh.j2 new file mode 100644 index 0000000000..324b646e72 --- /dev/null +++ b/roles/clamav/templates/clamscan.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/bash + +# Scans various directories for baddies +# Fri Jul 22 2011 athmane: modified to scan files older than $DAYS days + +LOGFILE="/var/log/clamscan.log" +MAILTO="{{ clamscan_mailto }}" +DAYS=7 +FILELIST="/tmp/clamscan_filelist.$$" + +rm -f $LOGFILE + +# Build file list to scan +{% for path in clamscan_paths %} +find {{ path }} -ctime -${DAYS} -type f >> $FILELIST +{% endfor %} + +/bin/nice -5 /usr/bin/clamscan --infected --quiet --file-list=$FILELIST --log=$LOGFILE \ +{% for path in clamscan_excludes|default([]) %} +--exclude={{ exclude }} \ +{% endfor %} +2> /dev/null + +RESULT=$? + +if [ "$RESULT" -eq "1" ] +then + mail -s "Virus found! on $(hostname)" $MAILTO -- < $LOGFILE +fi + +if [ "$RESULT" -eq "2" ] +then + mail -s "Virus scanning error on $(hostname)" $MAILTO -- < $LOGFILE +fi + +rm -f $FILELIST From 5132a5e46a4655aa48ab8650f0192d7e5d526ada Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 15 Jul 2014 12:09:12 +0200 Subject: [PATCH 121/193] Add a new gitolite/check_fedmsg_hooks role This will be needed to migrate Dist Git from puppet to ansible. --- roles/gitolite/check_fedmsg_hooks/tasks/main.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 roles/gitolite/check_fedmsg_hooks/tasks/main.yml diff --git a/roles/gitolite/check_fedmsg_hooks/tasks/main.yml b/roles/gitolite/check_fedmsg_hooks/tasks/main.yml new file mode 100644 index 0000000000..0d25329fdf --- /dev/null +++ b/roles/gitolite/check_fedmsg_hooks/tasks/main.yml @@ -0,0 +1,15 @@ +--- +# tasklist for setting up Gitolite Fedmsg checks + +- name: install the needed packages + yum: pkg=git state=present + +- name: install and schedule the script + copy: > + src=check-perms.py dest=/usr/local/bin/git-check-perms + owner=root group=root mode=0755 + cron: > + name="git-check-perms" cron_file="ansible-git-check-perms" + minute=10 hour=0 weekday=3 + user=nobody + job="MAILTO={{git_check_perms_mailto | "root"}} PATH=/usr/bin:/usr/local/bin git check-perms {{ git_basepath | "/git"}} --check=fedmsg-hook" From 7916770b41e87e3b618ef76f6af51a6ed951e031 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 15 Jul 2014 12:12:43 +0200 Subject: [PATCH 122/193] Add a new cgit/make_pkgs_list role This will be needed to migrate Dist Git from puppet to ansible. --- .../files/make-cgit-pkgs-list.sh | 13 +++++ roles/cgit/make_pkgs_list/tasks/main.yml | 53 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh create mode 100644 roles/cgit/make_pkgs_list/tasks/main.yml diff --git a/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh new file mode 100644 index 0000000000..21b1c921d2 --- /dev/null +++ b/roles/cgit/make_pkgs_list/files/make-cgit-pkgs-list.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# This simple script lists out the current pkgs git repos to a file. +# This speeds up cgit as it doesn't have to recurse into all dirs +# Looking for git repos. +# +newfile=`mktemp` + +cd /srv/git/rpms +ls > $newfile +mv $newfile /srv/git/pkgs-git-repos-list +chmod 644 /srv/git/pkgs-git-repos-list diff --git a/roles/cgit/make_pkgs_list/tasks/main.yml b/roles/cgit/make_pkgs_list/tasks/main.yml new file mode 100644 index 0000000000..fcf1032f90 --- /dev/null +++ b/roles/cgit/make_pkgs_list/tasks/main.yml @@ -0,0 +1,53 @@ +--- +# tasklist for setting up the CGit file list + +- name: install the script and schedule its execution + copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755 + cron: > + name="make-cgit-pkgs-list" cron_file="ansible-make-cgit-pkgs-list" + minute=*/10 + user=root + job="/usr/local/bin/lock-wrapper make-cgit-pkgs-list '/usr/local/bin/make-cgit-pkgs-list.sh | /usr/local/bin/nag-once fassync 1d 2>&1'" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 9dd33b21dd4891bc3535177f7d01a41093b96f1b Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Tue, 15 Jul 2014 12:22:52 +0200 Subject: [PATCH 123/193] Add a new git/make_checkout_seed role This will be needed to migrate Dist Git from puppet to ansible. --- .../files/make-git-checkout-seed.sh | 47 +++++++++++++++++++ roles/git/make_checkout_seed/tasks/main.yml | 13 +++++ 2 files changed, 60 insertions(+) create mode 100644 roles/git/make_checkout_seed/files/make-git-checkout-seed.sh create mode 100644 roles/git/make_checkout_seed/tasks/main.yml diff --git a/roles/git/make_checkout_seed/files/make-git-checkout-seed.sh b/roles/git/make_checkout_seed/files/make-git-checkout-seed.sh new file mode 100644 index 0000000000..e873523b90 --- /dev/null +++ b/roles/git/make_checkout_seed/files/make-git-checkout-seed.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Puppet notes -- script is file +# mkdir /srv/git_seed +# cron job to invoke file daily +# Need to setup OUTPUT_DIR to be served by apache + +# Where the git repos live. These are bare repos +ORIGIN_DIR=/srv/git/rpms + +# Where we'll create the repos to tar up +WORK_DIR=/srv/git_seed +# Subdirectory makes cleanup easier +SEED_DIR=$WORK_DIR/git-checkout + +# Where to store the seed tarball for download +OUTPUT_DIR=/srv/cache/lookaside/ + +# Instead of starting fresh each time, we'll try to use git pull to keep things synced +#rm -rf $WORK_DIR/* +mkdir -p $SEED_DIR + +# Give people an indication of when this checkout was last synced +TIMESTAMP=`date --rfc-3339=seconds` +DATE=`date +'%Y%m%d'` +echo "$TIMESTAMP" > $SEED_DIR/TIMESTAMP + + +for repo in $ORIGIN_DIR/*.git ; do + working_tree=$SEED_DIR/$(basename $repo .git) + if [ -d $working_tree ] ; then + pushd $working_tree &> /dev/null + sed -i "s@url = .*@url = $repo@" $working_tree/.git/config + git pull --all &> /dev/null + sed -i "s@url = .*@url = git://pkgs.fedoraproject.org/$(basename $repo .git)@" $working_tree/.git/config + popd &>/dev/null + else + pushd $SEED_DIR &>/dev/null + git clone $repo &> /dev/null + popd &>/dev/null + sed -i "s@url = .*@url = git://pkgs.fedoraproject.org/$(basename $repo .git)@" $working_tree/.git/config + fi +done + +tar -cf - -C$WORK_DIR $(basename $SEED_DIR)|xz -2 > $OUTPUT_DIR/.git-seed-$DATE.tar.xz +rm $OUTPUT_DIR/git-seed*tar.xz +mv $OUTPUT_DIR/.git-seed-$DATE.tar.xz $OUTPUT_DIR/git-seed-$DATE.tar.xz +ln -s git-seed-$DATE.tar.xz $OUTPUT_DIR/git-seed-latest.tar.xz diff --git a/roles/git/make_checkout_seed/tasks/main.yml b/roles/git/make_checkout_seed/tasks/main.yml new file mode 100644 index 0000000000..86a35bdeb9 --- /dev/null +++ b/roles/git/make_checkout_seed/tasks/main.yml @@ -0,0 +1,13 @@ +--- +# tasklist for setting up the Git checkout seed + +- name: create the destination directory + file: dest=/srv/git_seed owner=root group=root mode=0755 state=directory + +- name: install the script and schedule its execution + copy: src=make-git-checkout-seed.sh dest=/usr/local/bin/make-git-checkout-seed.sh mode=0755 + cron: > + name="make-git-checkout-seed" cron_file="ansible-make-git-checkout-seed" + minute=0 hour=2 + user=root + job='/usr/local/bin/lock-wrapper make-git-checkout-seed "/usr/local/bin/make-git-checkout-seed.sh | /usr/local/bin/nag-once fassync 1d 2>&1"' From f60221e2aa4d00a4e8e3d16a615eb46a011d24e6 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Thu, 17 Jul 2014 17:15:28 +0200 Subject: [PATCH 124/193] Add the confine_ssh task This is required by the drbackup stuff, which is required to migrate Dist Git to ansible. --- files/scripts/confine-ssh.sh | 16 ++++++++++++++++ tasks/confine_ssh.yml | 6 ++++++ 2 files changed, 22 insertions(+) create mode 100644 files/scripts/confine-ssh.sh create mode 100644 tasks/confine_ssh.yml diff --git a/files/scripts/confine-ssh.sh b/files/scripts/confine-ssh.sh new file mode 100644 index 0000000000..07a448fc46 --- /dev/null +++ b/files/scripts/confine-ssh.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Confine ssh commands +case "$SSH_ORIGINAL_COMMAND" in +*\&*) +echo "Rejected" +;; +*\;*) +echo "Rejected" +;; +rsync\ --server\ --sender*) +$SSH_ORIGINAL_COMMAND +;; +*) +echo "Rejected" +;; +esac diff --git a/tasks/confine_ssh.yml b/tasks/confine_ssh.yml new file mode 100644 index 0000000000..914d5ad1eb --- /dev/null +++ b/tasks/confine_ssh.yml @@ -0,0 +1,6 @@ +--- +- name: install the confine-ssh script + copy: > + src={{files}}/scripts/confine-ssh.sh + dest=/usr/local/bin/confine-ssh.sh + mode=0755 From 4392ef9f6cc655fe2434c54b05586d313851ef90 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Thu, 17 Jul 2014 18:25:32 +0200 Subject: [PATCH 125/193] Add a drbackupkey task This will be required to migrate Dist Git to ansible. --- tasks/drbackupkey.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tasks/drbackupkey.yml diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml new file mode 100644 index 0000000000..166230c580 --- /dev/null +++ b/tasks/drbackupkey.yml @@ -0,0 +1,13 @@ +--- +- name: ensure the user exists + group: name=drbackup state=present + user: name=drbackup comment="DR Backup User" group=drbackup shell=/bin/bash home=/var/lib/drbackup + file: dest=/var/lib/drbackup/ state=directory owner=drbackup group=drbackup mode=0700 + +- name: install the authorized SSH key + file: dest=/var/lib/drbackup/.ssh/ state=directory owner=drbackup group=drbackup mode=0700 + copy: src={{private}}/backup.pub dest=/var/lib/drbackup/.ssh/authorized_keys owner=drbackup group=drbackup mode=0600 + +- name: deploy the confine-ssh script + tasks: + - include: "{{tasks}}/confine-ssh.yml" From 30ab4bd52867a9244e72efe041404e6f396a895a Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Thu, 17 Jul 2014 18:35:17 +0200 Subject: [PATCH 126/193] Add a new distgit role This is the meat of Dist Git, but it requires quite a few of the roles previously created. --- roles/distgit/files/cgitrc | 71 ++ roles/distgit/files/fedmsg-genacls-config.py | 4 + roles/distgit/files/genacls.pkgdb | 118 ++++ roles/distgit/files/genacls.sh | 14 + roles/distgit/files/git-smart-http.conf | 3 + roles/distgit/files/gitolite.rc | 233 +++++++ roles/distgit/files/lookaside-upload.conf | 69 ++ roles/distgit/files/lookaside.conf | 6 + roles/distgit/files/mkbranch | 181 +++++ roles/distgit/files/pkgdb2-clone | 152 ++++ roles/distgit/files/pkgdb2branch.py | 362 ++++++++++ .../distgit/files/pkgs.fedoraproject.org.conf | 1 + roles/distgit/files/process-git-requests | 654 ++++++++++++++++++ roles/distgit/files/redirect.conf | 160 +++++ roles/distgit/files/setup_git_package | 125 ++++ roles/distgit/files/ssl.conf | 16 + roles/distgit/files/updatecrl.sh | 16 + roles/distgit/tasks/main.yml | 147 ++++ 18 files changed, 2332 insertions(+) create mode 100644 roles/distgit/files/cgitrc create mode 100644 roles/distgit/files/fedmsg-genacls-config.py create mode 100644 roles/distgit/files/genacls.pkgdb create mode 100644 roles/distgit/files/genacls.sh create mode 100644 roles/distgit/files/git-smart-http.conf create mode 100644 roles/distgit/files/gitolite.rc create mode 100644 roles/distgit/files/lookaside-upload.conf create mode 100644 roles/distgit/files/lookaside.conf create mode 100644 roles/distgit/files/mkbranch create mode 100644 roles/distgit/files/pkgdb2-clone create mode 100644 roles/distgit/files/pkgdb2branch.py create mode 100644 roles/distgit/files/pkgs.fedoraproject.org.conf create mode 100644 roles/distgit/files/process-git-requests create mode 100644 roles/distgit/files/redirect.conf create mode 100644 roles/distgit/files/setup_git_package create mode 100644 roles/distgit/files/ssl.conf create mode 100644 roles/distgit/files/updatecrl.sh create mode 100644 roles/distgit/tasks/main.yml diff --git a/roles/distgit/files/cgitrc b/roles/distgit/files/cgitrc new file mode 100644 index 0000000000..7276dc333b --- /dev/null +++ b/roles/distgit/files/cgitrc @@ -0,0 +1,71 @@ +# +# See cgitrc(5) or /usr/share/doc/cgit-*/cgitrc.5.html for details +# + +# Enable caching of up to 1000 output entries +cache-size=1000 + +# Specify some default clone prefixes +clone-prefix=git://pkgs.fedoraproject.org ssh://pkgs.fedoraproject.org http://pkgs.fedoraproject.org/git + +# Specify the css url +css=/cgit-data/cgit.css + +# Show extra links for each repository on the index page +enable-index-links=1 + +# Enable ASCII art commit history graph on the log pages +enable-commit-graph=1 + +# Show number of affected files per commit on the log pages +enable-log-filecount=1 + +# Show number of added/removed lines per commit on the log pages +enable-log-linecount=1 + +# Add a cgit favicon +#favicon=/favicon.ico + +# Use a custom logo +logo=/cgit-data/cgit.png + +# Enable statistics per week, month and quarter +max-stats=quarter + +# Set the title and heading of the repository index page +root-title=Fedora Project Packages GIT repositories + +# Set a subheading for the repository index page +#root-desc=tracking the foobar development + +# Include some more info about this site on the index page +#root-readme=/var/www/html/about.html + +# Allow download of tar.gz, tar.bz2 and zip-files +snapshots=tar.gz tar.xz zip + +## +## List of common mimetypes +## + +mimetype.gif=image/gif +mimetype.html=text/html +mimetype.jpg=image/jpeg +mimetype.jpeg=image/jpeg +mimetype.pdf=application/pdf +mimetype.png=image/png +mimetype.svg=image/svg+xml + +# Enable syntax highlighting (requires the highlight package) +#source-filter=/usr/libexec/cgit/filters/syntax-highlighting.sh +email-filter=lua:/usr/libexec/cgit/filters/email-libravatar-korg.lua + + +## +## List of repositories. +## PS: Any repositories listed when section is unset will not be +## displayed under a section heading +## PPS: This list could be kept in a different file (e.g. '/etc/cgitrepos') +## and included like this: +project-list=/srv/git/pkgs-git-repos-list +scan-path=/srv/git/rpms/ diff --git a/roles/distgit/files/fedmsg-genacls-config.py b/roles/distgit/files/fedmsg-genacls-config.py new file mode 100644 index 0000000000..b657de1de8 --- /dev/null +++ b/roles/distgit/files/fedmsg-genacls-config.py @@ -0,0 +1,4 @@ +config = { + 'genacls.consumer.enabled': True, + 'genacls.consumer.delay': 5, # 5 seconds +} diff --git a/roles/distgit/files/genacls.pkgdb b/roles/distgit/files/genacls.pkgdb new file mode 100644 index 0000000000..88f75e7970 --- /dev/null +++ b/roles/distgit/files/genacls.pkgdb @@ -0,0 +1,118 @@ +#!/usr/bin/python -t +# +# Create an /etc/gitolog/conf/getolog.conf file with acls for dist-git +# +# Takes no arguments! +# + +import grp +import sys + +import requests + +if __name__ == '__main__': + # Get the users in various groups + TRUSTED = grp.getgrnam('cvsadmin')[3] + ARM = grp.getgrnam('fedora-arm')[3] + SPARC = grp.getgrnam('fedora-sparc')[3] + IA64 = grp.getgrnam('fedora-ia64')[3] + S390 = grp.getgrnam('fedora-s390')[3] + PPC = grp.getgrnam('fedora-ppc')[3] + PROVEN = grp.getgrnam('provenpackager')[3] + + # Set the active branches to create ACLs for + # Give them the git branch eqiv until pkgdb follows suite + ACTIVE = {'OLPC-2': 'olpc2', 'OLPC-3': 'olpc3', 'EL-4': 'el4', + 'EL-5': 'el5', 'el5': 'el5', 'el6': 'el6', 'EL-6': 'el6', + 'epel7': 'epel7', + 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': + 'f15', 'f16': 'f16', 'f17': 'f17', 'f18': 'f18', 'f19': 'f19', + 'f20': 'f20', 'devel': 'master', 'master': 'master'} + + # Create a "regex"ish list 0f the reserved branches + RESERVED = ['f[0-9][0-9]', 'epel[0-9]', 'epel[0-9][0-9]', 'el[0-9]', 'olpc[0-9]'] + + # Read the ACL information from the packageDB + data = requests.get('https://admin.fedoraproject.org/pkgdb/api/vcs?format=json').json() + + # Get a list of all the packages + acls = data['packageAcls'] + pkglist = data['packageAcls'].keys() + pkglist.sort() + + # sanity check + if len(pkglist) < 2500: + sys.exit(1) + + # print out our user groups + print '@admins = %s' % ' '.join(TRUSTED) + print '@provenpackager = %s' % ' '.join(PROVEN) + print '@fedora-arm = %s' % ' '.join(ARM) + print '@fedora-s390 = %s' % ' '.join(S390) + print '@fedora-ppc = %s' % ' '.join(PPC) + + # print our default permissions + print 'repo @all' + print ' RWC = @admins @fedora-arm @fedora-s390 @fedora-ppc' + print ' R = @all' + #print ' RW private- = @all' + # dont' enable the above until we prevent building for real from private- + + for pkg in pkglist: + branchAcls = {} # Check whether we need to set separate per branch acls + buffer = [] # Buffer the output per package + masters = [] # Folks that have commit to master + writers = [] # Anybody that has write access + + # Examine each branch in the package + branches = acls[pkg].keys() + branches.sort() + for branch in branches: + if not branch in ACTIVE.keys(): + continue + if 'packager' in acls[pkg][branch]['commit']['groups']: + # If the packager group is defined, everyone has access + buffer.append(' RWC %s = @all' % (ACTIVE[branch])) + branchAcls.setdefault('@all', []).append((pkg, + ACTIVE[branch])) + if branch == 'master': + masters.append('@all') + if '@all' not in writers: + writers.append('@all') + else: + # Extract the owners + committers = [] + owners = acls[pkg][branch]['commit']['people'] + owners.sort() + for owner in owners: + committers.append(owner) + if 'provenpackager' in acls[pkg][branch]['commit']['groups']: + committers.append('@provenpackager') + if branch == 'master': + masters.extend(committers) + + # add all the committers to the top writers list + for committer in committers: + if not committer in writers: + writers.append(committer) + + # Print the committers to the acl for this package-branch + committers = ' '.join(committers) + buffer.append(' RWC %s = %s' % + (ACTIVE[branch], committers)) + branchAcls.setdefault(committers, []).append((pkg, + ACTIVE[branch])) + + print + print 'repo %s' % pkg + #if len(branchAcls.keys()) == 1: + # acl = branchAcls.keys()[0] + # print ' RW = %s' % acl + #else: + print '\n'.join(buffer) + for reserved in RESERVED: + print ' - %s = @all' % reserved + print ' RWC refs/tags/ = %s' % ' '.join(writers) + if masters: + print ' RWC = %s' % ' '.join(masters) + sys.exit(0) diff --git a/roles/distgit/files/genacls.sh b/roles/distgit/files/genacls.sh new file mode 100644 index 0000000000..712edb3479 --- /dev/null +++ b/roles/distgit/files/genacls.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +TEMPDIR=`mktemp -d -p /var/tmp genacls.XXXXX` +export GL_RC=/etc/gitolite/gitolite.rc +export GL_BINDIR=/usr/bin + +cd $TEMPDIR +# Only replace the acls if genacls completes successfully +if /usr/local/bin/genacls.pkgdb > gitolite.conf ; then + mv gitolite.conf /etc/gitolite/conf/ + /usr/bin/gl-compile-conf +fi +cd / +rm -rf $TEMPDIR diff --git a/roles/distgit/files/git-smart-http.conf b/roles/distgit/files/git-smart-http.conf new file mode 100644 index 0000000000..5841632b72 --- /dev/null +++ b/roles/distgit/files/git-smart-http.conf @@ -0,0 +1,3 @@ +SetEnv GIT_PROJECT_ROOT /srv/git/rpms +SetEnv GIT_HTTP_EXPORT_ALL +ScriptAlias /git/ /usr/libexec/git-core/git-http-backend/ diff --git a/roles/distgit/files/gitolite.rc b/roles/distgit/files/gitolite.rc new file mode 100644 index 0000000000..03149e3ce5 --- /dev/null +++ b/roles/distgit/files/gitolite.rc @@ -0,0 +1,233 @@ +# paths and configuration variables for gitolite + +# please read comments before editing + +# this file is meant to be pulled into a perl program using "do" or "require". + +# You do NOT need to know perl to edit the paths; it should be fairly +# self-explanatory and easy to maintain perl syntax :-) + +# -------------------------------------- +# Do not uncomment these values unless you know what you're doing +# $GL_PACKAGE_CONF = ""; +# $GL_PACKAGE_HOOKS = ""; + +# -------------------------------------- + +# -------------------------------------- + +# this is where the repos go. If you provide a relative path (not starting +# with "/"), it's relative to your $HOME. You may want to put in something +# like "/bigdisk" or whatever if your $HOME is too small for the repos, for +# example + +$REPO_BASE="/srv/git/rpms/"; + +# the default umask for repositories is 0077; change this if you run stuff +# like gitweb and find it can't read the repos. Please note the syntax; the +# leading 0 is required + +$REPO_UMASK = 0002; +# $REPO_UMASK = 0027; # gets you 'rwxr-x---' +# $REPO_UMASK = 0022; # gets you 'rwxr-xr-x' + +# part of the setup of gitweb is a variable called $projects_list (please see +# gitweb documentation for more on this). Set this to the same value: + +$PROJECTS_LIST = $ENV{HOME} . "/projects.list"; + +# -------------------------------------- + +# I see no reason anyone may want to change the gitolite admin directory, but +# feel free to do so. However, please note that it *must* be an *absolute* +# path (i.e., starting with a "/" character) + +# gitolite admin directory, files, etc + +$GL_ADMINDIR="/etc/gitolite"; + +# -------------------------------------- + +# templates for location of the log files and format of their names + +# I prefer this template (note the %y and %m placeholders) +# it produces files like `~/.gitolite/logs/gitolite-2009-09.log` + +$GL_LOGT="/var/log/gitolite/gitolite-%y-%m.log"; + +# other choices are below, or you can make your own -- but PLEASE MAKE SURE +# the directory exists and is writable; gitolite won't do that for you (unless +# it is the default, which is "$GL_ADMINDIR/logs") + +# $GL_LOGT="$GL_ADMINDIR/logs/gitolite-%y-%m-%d.log"; +# $GL_LOGT="$GL_ADMINDIR/logs/gitolite-%y.log"; + +# -------------------------------------- + +# Please DO NOT change these three paths + +$GL_CONF="$GL_ADMINDIR/conf/gitolite.conf"; +$GL_KEYDIR="$GL_ADMINDIR/keydir"; +$GL_CONF_COMPILED="$GL_ADMINDIR/conf/gitolite.conf-compiled.pm"; + +# -------------------------------------- + +# if git on your server is on a standard path (that is +# ssh git@server git --version +# works), leave this setting as is. Otherwise, choose one of the +# alternatives, or write your own + +$GIT_PATH=""; +# $GIT_PATH="/opt/bin/"; + +# -------------------------------------- + +# ---------------------------------------------------------------------- +# BIG CONFIG SETTINGS + +# Please read doc/big-config.mkd for details + +$GL_BIG_CONFIG = 1; +$GL_NO_DAEMON_NO_GITWEB = 1; +$GL_NO_CREATE_REPOS = 1; +$GL_NO_SETUP_AUTHKEYS = 1; + + +# ---------------------------------------------------------------------- +# SECURITY SENSITIVE SETTINGS +# +# Settings below this point may have security implications. That +# usually means that I have not thought hard enough about all the +# possible ways to crack security if these settings are enabled. + +# Please see details on each setting for specifics, if any. +# ---------------------------------------------------------------------- + + + +# -------------------------------------- +# ALLOW REPO ADMIN TO SET GITCONFIG KEYS +# +# Gitolite allows you to set git repo options using the "config" keyword; see +# conf/example.conf for details and syntax. +# +# However, if you are in an installation where the repo admin does not (and +# should not) have shell access to the server, then allowing him to set +# arbitrary repo config options *may* be a security risk -- some config +# settings may allow executing arbitrary commands. +# +# You have 3 choices. By default $GL_GITCONFIG_KEYS is left empty, which +# completely disables this feature (meaning you cannot set git configs from +# the repo config). + +$GL_GITCONFIG_KEYS = ""; + +# The second choice is to give it a space separated list of settings you +# consider safe. (These are actually treated as a set of regular expression +# patterns, and any one of them must match). For example: +# $GL_GITCONFIG_KEYS = "core\.logAllRefUpdates core\..*compression"; +# allows repo admins to set one of those 3 config keys (yes, that second +# pattern matches two settings from "man git-config", if you look) +# +# The third choice (which you may have guessed already if you're familiar with +# regular expressions) is to allow anything and everything: +# $GL_GITCONFIG_KEYS = ".*"; + +# -------------------------------------- +# EXTERNAL COMMAND HELPER -- HTPASSWD + +# security note: runs an external command (htpasswd) with specific arguments, +# including a user-chosen "password". + +# if you want to enable the "htpasswd" command, give this the absolute path to +# whatever file apache (etc) expect to find the passwords in. + +$HTPASSWD_FILE = ""; + +# Look in doc/3 ("easier to link gitweb authorisation with gitolite" section) +# for more details on using this feature. + +# -------------------------------------- +# EXTERNAL COMMAND HELPER -- RSYNC + +# security note: runs an external command (rsync) with specific arguments, all +# presumably filled in correctly by the client-side rsync. + +# base path of all the files that are accessible via rsync. Must be an +# absolute path. Leave it undefined or set to the empty string to disable the +# rsync helper. + +$RSYNC_BASE = ""; + +# $RSYNC_BASE = "/home/git/up-down"; +# $RSYNC_BASE = "/tmp/up-down"; + +# -------------------------------------- +# EXTERNAL COMMAND HELPER -- SVNSERVE + +# security note: runs an external command (svnserve) with specific arguments, +# as specified below. %u is substituted with the username. + +# This setting allows launching svnserve when requested by the ssh client. +# This allows using the same SSH setup (hostname/username/public key) for both +# SVN and git access. Leave it undefined or set to the empty string to disable +# svnserve access. + +$SVNSERVE = ""; +# $SVNSERVE = "/usr/bin/svnserve -r /var/svn/ -t --tunnel-user=%u"; + +# -------------------------------------- +# ALLOW REPO CONFIG TO USE WILDCARDS + +# security note: this used to in a separate "wildrepos" branch. You can +# create repositories based on wild cards, give "ownership" to the specific +# user who created it, allow him/her to hand out R and RW permissions to other +# users to collaborate, etc. This is powerful stuff, and I've made it as +# secure as I can, but it hasn't had the kind of rigorous line-by-line +# analysis that the old "master" branch had. + +# This has now been rolled into master, with all the functionality gated by +# this variable. Set this to 1 if you want to enable the wildrepos features. +# Please see doc/4-wildcard-repositories.mkd for details. + +$GL_WILDREPOS = 0; + +# -------------------------------------- +# DEFAULT WILDCARD PERMISSIONS + +# If set, this value will be used as the default user-level permission rule of +# new wildcard repositories. The user can change this value with the setperms command +# as desired after repository creation; it is only a default. Note that @all can be +# used here but is special; no other groups can be used in user-level permissions. + +# $GL_WILDREPOS_DEFPERMS = 'R = @all'; + +# -------------------------------------- +# HOOK CHAINING + +# by default, the update hook in every repo chains to "update.secondary". +# Similarly, the post-update hook in the admin repo chains to +# "post-update.secondary". If you're fine with the defaults, there's no need +# to do anything here. However, if you want to use different names or paths, +# change these variables + +# $UPDATE_CHAINS_TO = "hooks/update.secondary"; +# $ADMIN_POST_UPDATE_CHAINS_TO = "hooks/post-update.secondary"; + +# -------------------------------------- +# ADMIN DEFINED COMMANDS + +# WARNING: Use this feature only if (a) you really really know what you're +# doing or (b) you really don't care too much about security. Please read +# doc/admin-defined-commands.mkd for details. + +# $GL_ADC_PATH = ""; + +# -------------------------------------- +# per perl rules, this should be the last line in such a file: +1; + +# Local variables: +# mode: perl +# End: +# vim: set syn=perl: diff --git a/roles/distgit/files/lookaside-upload.conf b/roles/distgit/files/lookaside-upload.conf new file mode 100644 index 0000000000..a5948d7398 --- /dev/null +++ b/roles/distgit/files/lookaside-upload.conf @@ -0,0 +1,69 @@ +Alias /repo/ /srv/cache/lookaside/ + +# default SSL configuration... +Listen 443 + +SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000) +SSLSessionCacheTimeout 300 + +SSLMutex default + +SSLRandomSeed startup file:/dev/urandom 256 +SSLRandomSeed connect builtin +SSLCryptoDevice builtin + +# SSL host + + # This alias must come before the /repo/ one to avoid being overridden. + ScriptAlias /repo/pkgs/upload.cgi /srv/web/upload.cgi + + Alias /repo/ /srv/cache/lookaside/ + ServerName pkgs.fedoraproject.org + ServerAdmin webmaster@fedoraproject.org + + SSLEngine on + + SSLCertificateFile conf/pkgs.fedoraproject.org_key_and_cert.pem + SSLCertificateKeyFile conf/pkgs.fedoraproject.org_key_and_cert.pem + SSLCACertificateFile conf/cacert.pem + SSLCARevocationFile /etc/pki/tls/crl.pem + + SSLCipherSuite RSA:!EXPORT:!DH:!LOW:!NULL:+MEDIUM:+HIGH + +# Must be 'optional' everywhere in order to have POST operations work to upload.cgi + SSLVerifyClient optional +# Must be here for POST operations to upload.cgi + SSLOptions +OptRenegotiate + ErrorLog logs/ssl_error_log + CustomLog logs/ssl_access_log \ + "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%{SSL_CLIENT_S_DN_OU}x\" %{SSL_CLIENT_S_DN_CN}x %{SSL_CLIENT_S_DN_emailAddress}x \"%r\" %b" + + + SSLVerifyClient optional + SSLVerifyDepth 1 + SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate + # require that the client auth cert was created by us and signed by us + SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ + and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \ + and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \ + and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \ + and %{SSL_CLIENT_I_DN_OU} eq "Fedora Project CA" ) + + + + SSLRequireSSL + + SSLVerifyClient optional + SSLVerifyDepth 1 + SSLOptions +StrictRequire +StdEnvVars +OptRenegotiate + # require that the access comes from internal or that + # the client auth cert was created by us and signed by us + SSLRequire ( %{SSL_CIPHER} !~ m/^(EXP|NULL)/ \ + and %{SSL_CLIENT_S_DN_O} eq "Fedora Project" \ + and %{SSL_CLIENT_S_DN_OU} eq "Fedora User Cert" \ + and %{SSL_CLIENT_I_DN_O} eq "Fedora Project" \ + and %{SSL_CLIENT_I_DN_OU} eq "Fedora Project CA" ) + + + + diff --git a/roles/distgit/files/lookaside.conf b/roles/distgit/files/lookaside.conf new file mode 100644 index 0000000000..d7d3defccc --- /dev/null +++ b/roles/distgit/files/lookaside.conf @@ -0,0 +1,6 @@ +Alias /lookaside /srv/cache/lookaside + + Options Indexes FollowSymLinks + AllowOverride None + + diff --git a/roles/distgit/files/mkbranch b/roles/distgit/files/mkbranch new file mode 100644 index 0000000000..4e73495bdb --- /dev/null +++ b/roles/distgit/files/mkbranch @@ -0,0 +1,181 @@ +#!/bin/bash +# +# Create a new development branch for a module. +# THIS HAS TO BE RUN ON THE GIT SERVER! + +# WARNING: +# This file is maintained within puppet? +# All local changes will be lost. + + +# Figure out the environment we're running in +RUNDIR=$(cd $(dirname $0) && pwd) +GITROOT=/srv/git/rpms + +# check if a moron is driving me +if [ ! -d $GITROOT ] ; then + # we're not on the git server (this check is fragile) + echo "ERROR: This script has to be run on the git server." + echo "ERROR: Homer sez 'Duh'." + exit -9 +fi + +# where are the packages kept +TOPLEVEL=rpms + +# Local variables +VERBOSE=0 +TEST= +IGNORE= +BRANCH="" +PACKAGES="" +SRC_BRANCH="master" +AUTHOR="Fedora Release Engineering " + +Usage() { + cat <] ... + + Creates a new branch for the list of s. + The /master suffix on branch names is assumed. + +Options: + -s,--source= Use as the source branch. + Defaults is master + /master suffix on other branches assumed + -n,--test Don't do nothing, only test + -i,--ignore Ignore erroneous modules + -h,--help This help message + -v,--verbose Increase verbosity +EOF +} + +# parse the arguments +while [ -n "$1" ] ; do + case "$1" in + -h | --help ) + Usage + exit 0 + ;; + + -v | --verbose ) + VERBOSE=$(($VERBOSE + 1)) + ;; + + -i | --ignore ) + IGNORE="yes" + ;; + + -n | --test ) + TEST="yes" + ;; + + -s | --source ) + shift + SRC_BRANCH=$1 + ;; + + -b | --branch ) + shift + BRANCH=$1/master + ;; + + * ) + if [ -z "$BRANCH" ] ; then + BRANCH="$1" + else + PACKAGES="$PACKAGES $1" + fi + ;; + esac + shift +done + +# check the arguments +if [ -z "$BRANCH" -o -z "$PACKAGES" ] ; then + Usage + exit -1 +fi + + +# Sanity checks before we start doing damage +NEWP= +for p in $PACKAGES ; do + [ $VERBOSE -gt 1 ] && echo "Checking package $p..." + if [ ! -d $GITROOT/$p.git ] ; then + echo "ERROR: Package module $p is invalid" >&2 + [ "$IGNORE" = "yes" ] && continue || exit -1 + fi + if [ -z "$(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify $SRC_BRANCH)" ] ; then \ + echo "ERROR: Invalid source branch '$SRC_BRANCH' for package $p" >&2; \ + if [ $SRC_BRANCH == 'master' ]; then + [ "$IGNORE" = "yes" ] && continue + else + SRC_BRANCH=master + fi + fi + $(GIT_DIR=$GITROOT/$p.git git rev-parse -q --verify \ + $BRANCH >/dev/null) && \ + (echo "IGNORING: Package module $p already has a branch $BRANCH" >&2; \ + [ "$IGNORE" = "yes" ] && continue || exit -1) + NEWP="$NEWP $p" +done +PACKAGES="$(echo $NEWP)" +if [ -z "$PACKAGES" ] ; then + echo "NOOP: no valid packages found to process" + exit -1 +fi + +if [ -n "$TEST" ] ; then + echo "Branch $BRANCH valid for $PACKAGES" + exit 0 +fi + +# This account must have the proper permissions as to not screw up the +# repository work. +if [ "$(id -un)" = "root" ] ; then + echo "Please run this script as yourself" + exit -3 +fi +#### Change this to check for proper git-admin rights + +# "global" permissions check +if [ ! -w $GITROOT ] ; then + echo "ERROR: You can not write to $GITROOT" + echo "ERROR: You can not perform branching operations" + exit -1 +fi + +# Now start working on creating those branches + +# For every module, "create" the branch +for NAME in $PACKAGES ; do + echo + echo "Creating new module branch '$BRANCH' for '$NAME' from branch '$SRC_BRANCH'..." + + # permissions checks for this particular module + if [ ! -w $GITROOT/$NAME.git/refs/heads/ ] ; then + echo "ERROR: You can not write to $d" + echo "ERROR: $NAME can not be branched by you" + continue + fi + #### Replace the above with a gitolite permission check + #[ $VERBOSE -gt 0 ] && echo "Creating $BRANCH-split tag for $NAME/$SRC_BRANCH..." + # Is the above needed? + #cvs -Q rtag -f "$BRANCH-split" $TOPLEVEL/$NAME/$SRC_BRANCH || { + #echo "ERROR: Branch split tag for $NAME/$SRC_BRANCH could not be created" >&2 + #exit -2 + #} + [ $VERBOSE -gt 0 ] && echo "Creating $NAME $BRANCH from $NAME $SRC_BRANCH..." + $(pushd $GITROOT/$NAME.git >/dev/null && \ + git branch --no-track $BRANCH $SRC_BRANCH && \ + popd >/dev/null) || { + echo "ERROR: Branch $NAME $BRANCH could not be created" >&2 + popd >/dev/null + exit -2 + } +done + +echo +echo "Done." diff --git a/roles/distgit/files/pkgdb2-clone b/roles/distgit/files/pkgdb2-clone new file mode 100644 index 0000000000..49c8a8f736 --- /dev/null +++ b/roles/distgit/files/pkgdb2-clone @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import re +import requests +import sys +import getpass +import pkgdb2client +import subprocess + +#PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=User:Codeblock/RequestsSANDBOX' +PAGE_URL = 'https://fedoraproject.org/w/api.php?format=json&action=query&rvprop=content&prop=revisions&titles=EPEL/epel7/Requests' +NEW_EPEL_VERSION = '7' +NEW_EPEL_SOURCE_BRANCH = 'f19' +RHEL_PKGS_PATH = '/var/lib/rhel/rhel' + NEW_EPEL_VERSION + +# parse_page :: String -> IO (Map String String) +# This returns a dictionary of {"pkg_name": "branch"} +def parse_page(url): + r = requests.get(url).json() + text = r['query']['pages'][r['query']['pages'].keys()[0]]['revisions'][0]['*'] + lines = text.split("\n") + pkgs = filter(lambda y: y.startswith('| '), lines) + __pkgs_list__ = map(lambda y: ''.join(y.split())[1:].split('||'), pkgs) + pkgs_list = filter(lambda y: y[0] != 'foo', __pkgs_list__) + pkgs_dict = dict(pkgs_list) + return pkgs_dict + +# is_in_rhel :: String -> IO Bool +def is_in_rhel(pkg): + with open(RHEL_PKGS_PATH) as f: + pkgs = map(lambda x: x.strip(), f.readlines()) + return (pkg in pkgs) + +# These tuples will be used to substitute one pattern for another. +# Every transform will be run on every branch name so be sure the +# pattern cannot match if you don't want it to be triggered. +transforms = ( + (re.compile(r'^devel$'), 'master'), + (re.compile(r'-'), ''), + (re.compile(r'^fc([0-9])'), r'f\1'), + (re.compile(r'^epel([456])$'), r'el\1'), + (re.compile(r'^el([789]|[1-9][0-9])'), r'epel\1'), +) +branch_replacements = {'devel': (re.compile(r'^devel$'), 'master'),} + +# generate_collection_cache :: PkgDB -> IO [String] +def generate_collection_cache(pkgdb): + raw_collections = pkgdb.get_collections(clt_status=( + 'Active', + 'Under Development')) + collection_cache = frozenset(map(lambda y: y['branchname'], + raw_collections['collections'])) + return collection_cache + +# normalize_branch :: [String] -> String -> IO (Option String) +def normalize_branch(collection_cache, branch): + # I originally had this implemented as a foldRight (which it really is). + # But Python doesn't eliminate tail calls. It probably would have been fine + # because "transforms" above is only 5 elements, but instead I will deal + # with the local mutation and wish that I had a type system to reason with. + # -rbe + norm_branch = branch.lower() + for transform in transforms: + norm_branch = re.sub(transform[0], transform[1], norm_branch) + + + # Ugh, here we break purity. Where is the option type when you need it? + if not (norm_branch in collection_cache): + print('Unknown collection specified: {0}'.format(branch)) + return None + + return norm_branch + +# process_package :: PkgDB -> String -> String -> IO Bool +def process_package(pkgdb, pkg, src, dest): + data = pkgdb.get_package(pkg) + pkg_list = data['packages'] + + maybe_source = filter(lambda y: y['collection']['branchname'] == src, + pkg_list) + maybe_dest = filter(lambda y: y['collection']['branchname'] == dest, + pkg_list) + if len(maybe_source) == 0: + print "Source branch `" + src + "' not found. Please "\ + "branch" + pkg + "manually." + return False + + if len(maybe_dest) != 0: + print "Package `" + pkg + "' was already branched for `" + dest + "'."\ + " Not overwriting branch." + return False + + acls = filter(lambda y: y['fas_name'] != 'group::provenpackager', + maybe_source[0]['acls']) + map(lambda acl: pkgdb.update_acl(pkg, dest, acl['acl'], acl['status'], + acl['fas_name']), acls) + return True + +# main :: [String] -> IO Unit +def main(args): + new_epel_requests = "epel" + NEW_EPEL_VERSION + "-requests" + if len(args) < 1 or (len(args) < 3 and args[0] != new_epel_requests) or\ + len(args) > 3 or (len(args) > 1 and args[0] == new_epel_requests): + print "Usage: pkgdb2-clone " + new_epel_requests + print " - OR -" + print " pkgdb2-clone " + sys.exit(1) + + pkgdb = pkgdb2client.PkgDB() + + username = raw_input('Username: ') + password = getpass.getpass() + pkgdb.login(username, password, True) + + collection_cache = generate_collection_cache(pkgdb) + + if args[0] == new_epel_requests: + pkgs = parse_page(PAGE_URL) + for key in pkgs: + if is_in_rhel(key): + continue + src_branchname = normalize_branch(collection_cache, pkgs[key]) + dest_branchname = normalize_branch(collection_cache, + 'epel' + NEW_EPEL_VERSION) + if not src_branchname or not dest_branchname: + print "[" + key + "] Invalid source or destination branch "\ + "name, " + src_branchname + " -> " + dest_branchname + else: + if process_package(pkgdb, key, src_branchname, dest_branchname): + subprocess.call(["mkbranch", + "-s", + NEW_EPEL_SOURCE_BRANCH, + "epel" + NEW_EPEL_VERSION, + key]) + print "[" + key + "] Success" + else: + print "[" + key + "] Error" + print "Done." + else: + src_branchname = normalize_branch(collection_cache, args[0]) + dest_branchname = normalize_branch(collection_cache, args[1]) + if not src_branchname or not dest_branchname: + print "[" + key + "] Invalid source or destination branch "\ + "name, " + src_branchname + " -> " + dest_branchname + for pkg in args[2:]: + if process_package(pkgdb, key, src_branchname, dest_branchname): + print "[" + key + "] Success" + else: + print "[" + key + "] Error" + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/roles/distgit/files/pkgdb2branch.py b/roles/distgit/files/pkgdb2branch.py new file mode 100644 index 0000000000..507fdd120b --- /dev/null +++ b/roles/distgit/files/pkgdb2branch.py @@ -0,0 +1,362 @@ +#!/usr/bin/python -t +# Author: Toshio Kuratomi +# Copyright: 2007-2008 Red Hat Software +# License: GPLv2 +# This needs a proper license and copyright here +__version__ = '0.3' + +import sys +import os +import optparse + +import subprocess + +import fedmsg + +# Do some off-the-bat configuration of fedmsg. +# 1) since this is a one-off script and not a daemon, it needs to connect to +# the fedmsg-relay process running on another node (or noone will hear it) +# 2) its going to use the 'shell' certificate which only 'sysadmin' has read +# access to. Contrast that with the 'scm' certificate which everyone in +# the 'packager' group has access to. +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) + +from fedora.client import FedoraServiceError +from pkgdb2client import PkgDB + +GITDIR='/srv/git/rpms' +BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/' +MKBRANCH='/usr/local/bin/mkbranch' +SETUP_PACKAGE='/usr/local/bin/setup_git_package' +BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'master', 'epel7': 'f19', + 'olpc2': 'f7', + 'olpc3': 'f11', + 'master': None, + 'fc6': 'master', + 'f7': 'master', + 'f8': 'master', + 'f9': 'master', + 'f10': 'master', + 'f11': 'master', + 'f12': 'master', + 'f13': 'master', 'f14': 'master', + 'f15': 'master', 'f16': 'master', + 'f17': 'master', 'f18': 'master', + 'f19': 'master', 'f20': 'master' + } + +# The branch names we get out of pkgdb have to be translated to git +GITBRANCHES = {'el4': 'el4', 'el5': 'el5', 'el6': 'el6', 'epel7': 'epel7', + 'OLPC-2': 'olpc2', + 'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9', + 'F-10': 'f10', 'OLPC-3': 'olpc3', + 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15', 'f16': 'f16', 'f17': 'f17', + 'f18': 'f18', 'f19': 'f19', 'f20': 'f20', + 'devel': 'master'} + +# The branch options we get from the CLI have to be translated to pkgdb +BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()]) + +class InternalError(Exception): + pass + +class PackageDBError(InternalError): + pass + +class ProcessError(InternalError): + pass + +class ArgumentsError(InternalError): + pass + +class InvalidBranchError(PackageDBError): + pass + +class PackageDBClient(PkgDB): + def __init__(self, baseURL): + '''Initialize the connection. + + Args: + :baseURL: URL from which the packageDB is accessed + ''' + # We're only performing read operations so we don't need a username + super(PackageDBClient, self).__init__(baseURL) + + def get_package_branches(self, pkgname): + '''Return the branches to which a package belongs. + + Args: + :pkgname: The package to retrieve branch information about + ''' + + data = self.get_package(pkgname) + return map(lambda x: x['collection']['branchname'], data['packages']) + + def get_package_list(self, branchName): + '''Retrieve all the packages in a specific branch. + + Args: + :branchName: to return the packages for + ''' + pkgs = map(lambda l: l['name'], self.get_packages('*', branchName, page=0)['packages']) + return pkgs + +class Brancher(object): + ''' Make branches in the GIT Repository.''' + + def __init__(self, pkgdburl, cache, verbose): + # Connect to the package database + self.verbose = verbose + self.client = PackageDBClient(BASEURL) + + def _invoke(self, program, args): + '''Run a command and raise an exception if an error occurred. + + Args: + :program: The program to invoke + :args: List of arguments to pass to the program + + raises ProcessError if there's a problem. + ''' + cmdLine = [program] + cmdLine.extend(args) + print ' '.join(cmdLine) + + stdoutfd = subprocess.PIPE + if self.verbose: + program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) + else: + program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + retCode = program.wait() + if retCode != 0: + e = ProcessError() + e.returnCode = retCode + e.cmd = ' '.join(cmdLine) + if self.verbose: + output = program.stdout.read() + e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output) + else: + e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) + raise e + + def _create_branch(self, pkgname, branch): + '''Create a specific branch for a package. + + Args: + :pkgname: Name of the package to branch + :branch: Name of the branch to create + + raises InvalidBranchError if a branchname is unknown. + + Will ignore a branch which is EOL. + ''' + try: + branchFrom = '%s/master' % BRANCHES[branch] + except KeyError: + raise InvalidBranchError( + 'PackageDB returned an invalid branch %s for %s' % + (branch, pkgname)) + + # Add the master to the branch + # No longer add this after the new branching setup. + #branch = '%s/master' % branch + # If branchFrom is None, this is an EOL release + # If the directory already exists, no need to invoke mkbranch + if branchFrom: + # Fall back to branching from master. + frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname, + 'refs/heads', branchFrom) + if not os.path.exists(frombranchpath): + branchFrom = 'master' + + branchpath = os.path.join(GITDIR, '%s.git' % pkgname, + 'refs/heads', branch) + if not os.path.exists(branchpath): + try: + self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname]) + except ProcessError, e: + if e.returnCode == 255: + # This is a warning, not an error + return + raise + finally: + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent=os.getlogin(), + name=pkgname, + branch=branch, + ), + ) + + def branch_package(self, pkgname): + '''Create all the branches that are listed in the pkgdb for a package. + + Args: + :pkgname: The package to create branches for + + Note: this will ignore branches which are EOL. + + raises PackageDBError if the package is not present in the Package + Database. + ''' + # Retrieve branch information + try: + branches = self.client.get_package_branches(pkgname) + except FedoraServiceError, e: + raise PackageDBError( + 'Unable to retrieve information about %s: %s' % + (pkgname, str(e))) + + # Create the devel branch if necessary + if not os.path.exists(os.path.join(GITDIR, + '%s.git' % pkgname)): + self._invoke(SETUP_PACKAGE, [pkgname]) + # Create all the required branches for the package + # Use the translated branch name until pkgdb falls inline + for branch in branches: + if branch == 'devel': + continue + if not branch in GITBRANCHES.keys(): + print 'Skipping unknown branch %s' % branch + continue + self._create_branch(pkgname, GITBRANCHES[branch]) + + def mass_branch(self, branchName): + '''Make sure all packages listed for a specific branch in the PackageDB + have a CVS branch. + + Args: + :branchName: The branch to ensure. + ''' + fedmsg.publish( + topic='mass_branch.start', + modname='git', + msg=dict(agent=os.getlogin()), + ) + # Retrieve all the packages in this branch + pkglist = self.client.get_package_list(branchName) + pkglist.sort() + for pkg in pkglist: + # Create a branch for this release for each of them + # Use the translated branch name until pkgdb falls inline + self._create_branch(pkg, GITBRANCHES[branchName]) + + fedmsg.publish( + topic='mass_branch.complete', + modname='git', + msg=dict(agent=os.getlogin()), + ) + +def parse_commands(): + parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-] + pkgdb2branch.py [options] --branchfor BRANCH + +pkgdb2branch reads package information from the packagedb and creates branches +on the git server based on what branches are listed there. pkgdb2branch can +read the list of packages from stdin if you specify '-' as an argument. + +pkgdb2branch has two modes of operation. In the first mode, you specify which +packages you want to branch. This mode is more efficient for a small number +of packages. + +In the second mode, pkgdb2branch will find every package lacking a BRANCH and +will create one if the pkgdb says it's needed. This mode is very efficient for +mass branching. This implies --cache-branches. + +For those with a moderate number of packages, using a list of packages and +--cache-branches may be fastest.''') + parser.add_option('-b', '--branch-for', + dest='branchFor', + action='store', + help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.') + parser.add_option('-c', '--cache-branches', + dest='enableCache', + action='store_true', + help='Download a complete cache of packages') + parser.add_option('--verbose', + dest='verbose', + action='store_true', + help='Enable verbose output') + (opts, args) = parser.parse_args() + + if opts.branchFor: + if args: + raise ArgumentsError('Cannot specify packages with --branchFor') + opts.enableCache = True + + if '-' in args: + opts.fromStdin = True + del (args[args.index('-')]) + else: + opts.fromStdin = False + + if not (args or opts.fromStdin or opts.branchFor): + raise ArgumentsError('You must list packages to operate on') + + return opts, args + +if __name__ == '__main__': + try: + options, packages = parse_commands() + except ArgumentsError, e: + print e + sys.exit(1) + + branchedPackages, unbranchedPackages = [], [] + brancher = Brancher(BASEURL, options.enableCache, options.verbose) + fedmsg.publish( + topic='pkgdb2branch.start', + modname='git', + msg=dict(agent=os.getlogin()), + ) + + if options.branchFor: + try: + unbranchedPackages = \ + brancher.mass_branch(BRANCHBYGIT[options.branchFor]) + except PackageDBError, e: + print 'Unable contact the PackageDB. Error: %s' % str(e) + sys.exit(1) + else: + # Process packages specified on the cmdline + for pkgname in packages: + try: + brancher.branch_package(pkgname) + branchedPackages.append(pkgname) + except InternalError, e: + print str(e) + unbranchedPackages.append(pkgname) + + # Process packages from stdin + if options.fromStdin: + for pkgname in sys.stdin: + pkgname = pkgname.strip() + try: + brancher.branch_package(pkgname) + branchedPackages.append(pkgname) + except InternalError, e: + print str(e) + unbranchedPackages.append(pkgname) + + fedmsg.publish( + topic='pkgdb2branch.complete', + modname='git', + msg=dict( + agent=os.getlogin(), + branchedPackages=branchedPackages, + unbranchedPackages=unbranchedPackages, + ), + ) + + if unbranchedPackages: + print 'The following packages were unbranched:' + print '\n'.join(unbranchedPackages) + sys.exit(100) + + sys.exit(0) diff --git a/roles/distgit/files/pkgs.fedoraproject.org.conf b/roles/distgit/files/pkgs.fedoraproject.org.conf new file mode 100644 index 0000000000..cf4e0181c3 --- /dev/null +++ b/roles/distgit/files/pkgs.fedoraproject.org.conf @@ -0,0 +1 @@ +include "conf.d/pkgs.fedoraproject.org/*.conf" diff --git a/roles/distgit/files/process-git-requests b/roles/distgit/files/process-git-requests new file mode 100644 index 0000000000..c2bb9b2d4b --- /dev/null +++ b/roles/distgit/files/process-git-requests @@ -0,0 +1,654 @@ +#!/usr/bin/python -t +VERSION = "1.0" + +# $Id: process-cvs-requests.py,v 1.25 2010/07/01 04:51:23 tibbs Exp $ + +# TODO: +# Extract fedora-review flag setter if possible. +# Display last linked spec file. +# Download (and process?) last linked srpm + +# Checks to add: +# Package already exists in pkgdb. +# fedora-review flag isn't set (especially if it's still set to '?' +# Catch common misspellings? +# Any owner contains '@' or other invalid character +# Maybe verify owners in pkgdb/FAS. +# SSH into cvs.fedoraproject.org to run pkg2branch.py directly +# or just run on cvs.fedoraproject.org +# Try to do some checking on the ~/.bugzillacookies file and suggest "bugzilla login" + +import bugzilla +import codecs +import datetime +import getpass +import glob +import operator +import os +import re +import readline +import sys +import subprocess +import tempfile +import time +import xmlrpclib +from configobj import ConfigObj, flatten_errors +from fedora.client import AuthError, AppError, PackageDB +from optparse import OptionParser +from validate import Validator + +# Red Hat's bugzilla +url = 'https://bugzilla.redhat.com/xmlrpc.cgi' + +# Users who indicated that they're OK with EPEL branches. Some request that +# they be made comaintainers. +# Taken from http://fedoraproject.org/wiki/EPEL/ContributorStatusNo +epel_ok = ['abompart', 'athimm', 'corsepiu', 'ecik', 'faucamp', 'konradm', + 'monnerat', 'mtasaka', 'nim', 'rafalzaq', 'rineau', 'rstrode', + 'sgrubb', 'shishz', 'terjeros', 'zkota'] +epel_ok_comaint = ['alexlan', 'guidograzioli', 'jwrdegoede', 'kkofler', + 'mebourne', 'overholt', 'pgordon', 'rishi', 'snirkel'] + +PAGER = os.environ.get('PAGER') or '/usr/bin/less' +EDITOR = os.environ.get('EDITOR') or '/usr/bin/vim' + +# Override a method in xmlrpclib so it doesn't blow up when getting crap data +# from Red Hat's bugzilla. +# Bugfixes seem to have rendered this unnecessary +#def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search): +# # decode non-ascii string (if possible) +# if unicode and encoding and is8bit(data): +# data = unicode(data, encoding, 'replace') +# return data +#xmlrpclib._decode = _decode + +def parse_commandline(): + usage = 'usage: %prog [options]' + parser = OptionParser(usage) + parser.add_option('--url', dest='url', + help='bugzilla URL to query', + default=url) + parser.add_option('-u', '--user', + help='Username for PackageDB connection', + dest='user', + default=getpass.getuser()) + parser.add_option('--debug', + action='store_true', + dest='debug', + default=False, + help='Turn on some debugging statements') + + (options, args) = parser.parse_args() + return options + +def parse_pkgdb_config(): + vldtr = Validator() + # configspec to validate types and set defaults + configspec = ''' + [global] + pkgdb.url = string(default = 'https://admin.fedoraproject.org/pkgdb') + pkgdb.retries = integer(default = 5) + pkgdb.knowngroups = list(default = list()) + '''.splitlines() + + cfg = ConfigObj('/etc/pkgdb-client.cfg', configspec=configspec) + user = ConfigObj(os.path.expanduser('~/.fedora/pkgdb-client.cfg'), + configspec=configspec) + cfg.merge(user) + res = cfg.validate(vldtr, preserve_errors=True) + + for entry in flatten_errors(cfg, res): + section_list, key, error = entry + section_list.append(key) + section_string = ','.join(section_list) + if error == False: + error = 'Missing value or section.' + print ','.join(section_list), '=', error + sys.exit(1) + + cfg['global']['pkgdb.url'] = os.environ.get('PACKAGEDBURL') or cfg['global']['pkgdb.url'] + return cfg['global'] + +def encode_utf8(object, encoding='utf8', errors='replace'): + if isinstance(object, basestring): + if isinstance(object, str): + return unicode(object, encoding, errors) + else: + return object + return u'' + +def add_package(pkgdb, request): + for retry in range(1, config['pkgdb.retries'] + 1): + try: + pkgdb.add_package(pkg=request['pkg'], + owner=request['owner'], + description=request['description'], + branches=request['branches'], + cc_list=request['cc_list'], + comaintainers=request['comaintainers']) + except AuthError, e: + if sys.stdin.isatty(): + if retry >= config['pkgdb.retries']: + break + pkgdb.password = getpass.getpass('PackageDB Password: ') + else: + # Don't retry if we're reading the password from stdin + break + else: + break + +def edit_package(pkgdb, request): + for retry in range(1, config['pkgdb.retries'] + 1): + try: + pkgdb.edit_package(pkg=request['pkg'], + owner=request['owner'], + branches=request['newbranches'], + cc_list=request['cc_list'], + comaintainers=request['comaintainers']) + except AuthError, e: + if retry >= config['pkgdb.retries']: + break + pkgdb.password = getpass.getpass('PackageDB Password: ') + else: + break + +def run_query(bz): + querydata = {} + querydata['column_list'] = ['opendate', 'changeddate', 'bug_severity', + 'alias', 'assigned_to', 'reporter', 'bug_status', 'resolution', + 'component', 'blockedby', 'dependson', 'short_desc', + 'status_whiteboard', 'flag_types'] + querydata['product'] = ['Fedora'] + + querydata['field0-0-0'] = 'flagtypes.name' + querydata['type0-0-0'] = 'equals' + querydata['value0-0-0'] = 'fedora-cvs?' + + bugs = bz.query(querydata) + bugs.sort(key=operator.attrgetter('bug_id')) + + ids = map(lambda x: x.bug_id, bugs) + comments = bz._proxy.Bug.comments({"ids": ids}) + + return [bugs, comments] + +def display_bug(bug, comments): + '''Show the complete ticket in a pager.''' + comment = 0 + b = [] + b.append('https://bugzilla.redhat.com/%d' % bug.bug_id) + b.append('Bug %d - %s' % (bug.bug_id, bug.short_desc)) + b.append('Reported by: %s at %s' % (bug.reporter, bug.opendate)) + b.append('Assigned to: %s' % (bug.assigned_to)) + for i in comments: + b.append('-'*40) + #b.append('Comment %d by %s at %s\n' % (comment, i['author'], time.strftime('%F %T',i['time'].timetuple()))) + #b.append('Comment %d by %s at %04d-%02d-%02d %02d:%02d%02d\n' % ( + b.append('Comment %d by %s at %s\n' % ( + comment, i['author'], i['time'])) + b.append(i['text']) + b.append('') + comment += 1 + + p = subprocess.Popen(PAGER, stdin=subprocess.PIPE) + p.communicate('\n'.join(b).encode('utf8')) + + +def edit_string(s): + '''Edit the contents of a string in the user's preferred editor.''' + (fd, f) = tempfile.mkstemp() + fh=os.fdopen(fd, 'w+') + fh.write(s) + fh.close() + p = subprocess.Popen([EDITOR, f]); + sts = os.waitpid(p.pid, 0)[1] + if not sts: + try: + fh = open(f, 'r') + s = fh.read() + finally: + fh.close() + + return s + + +def parse_prefixed_lines(s): + lastitem = '' + items = {} + items['Branches'] = '' + lines = s.splitlines() + + # Skip until the Request line + while 1: + if (lines[0].find('New Package CVS Request') == 0 + or lines[0].find('Package Change Request') == 0): + break + lines.pop(0) + + # Skip until a line containing a colon + while 1: + if lines[0].find(':') >= 0: + break + lines.pop(0) + + # Now parse + while 1: + if not len(lines): + break + + line = lines.pop(0) + line.strip() + if len(line) == 0: + break + + pos = line.find(':') + + # Line-wrapped? + if pos < 0: + items[lastitem] += " " + line.strip() + continue + + lastitem = line[:pos] + items[lastitem] = line[pos+1:].strip() + + return items + +def clean_request(items): + '''Clean up various bits that can be passed in a CVS request.''' + request = {} + + if not 'InitialCC' in items: + items['InitialCC'] = '' + if not 'Owners' in items: + items['Owners'] = '' + if not 'Short Description' in items: + items['Short Description'] = '' + + branches = items['Branches'].strip() + branches = re.sub(r',', ' ', branches) + branches = re.sub(r'f', 'F', branches) + branches = re.sub(r'devel', ' ', branches) + branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches) + branches = re.sub(r'EL([1-9])', r'EL-\1', branches) + branches = re.sub(r'F-14', r'f14', branches) + branches = re.sub(r' +', ' ', branches) + branches = branches.strip() + branches += ' devel' + items['Branches'] = branches + request['branches'] = branches.split() + + if 'New Branches' in items: + branches = items['New Branches'].strip() + branches = re.sub(r',', ' ', branches) + branches = re.sub(r'f', 'F', branches) + branches = re.sub(r'F([1-9][0-9])', r'F-\1', branches) + branches = re.sub(r'F-14', r'f14', branches) + branches = re.sub(r' +', ' ', branches) + branches = branches.strip() + items['New Branches'] = branches + request['newbranches'] = branches.split() + + owners = items['Owners'].strip() + owners = re.sub(r',', ' ', owners) + if len(owners): + request['owner'] = owners.split()[0] + request['comaintainers'] = owners.split()[1:] + + request['cc_list'] = items['InitialCC'].split() + request['pkg'] = items['Package Name'] + request['description'] = items['Short Description'] + + return request + +def new_request_string(items, bug): + r = [] + r.append("Bug URL: http://bugzilla.redhat.com/%d " % bug.bug_id) + r.append("Bug summary: " + bug.short_desc) + r.append('') + r.append("New Package CVS Request") + r.append("=======================") + r.append("Package Name: " + items['Package Name']) + r.append("Short Description: " + items['Short Description']) + r.append("Owners: " + items['Owners']) + r.append("Branches: " + items['Branches']) + r.append("InitialCC: " + items['InitialCC']) + r.append('') + return '\n'.join(r) + +def change_request_string(items, bug): + r = [] + r.append("Bug URL: http://bugzilla.redhat.com/%d" % bug.bug_id) + r.append("Bug summary: " + bug.short_desc) + r.append('') + r.append("Package Change Request") + r.append("======================") + r.append("Package Name: " + items['Package Name']) + r.append("Owners: " + items['Owners']) + r.append("New Branches: " + items['New Branches']) + r.append("InitialCC: " + items['InitialCC']) + r.append('') + return '\n'.join(r) + +def get_pkgdb_owners(pkgdb, pkg): + owners = {} + o = '' + for i in pkgdb.get_owners(pkg)['packageListings']: + branch = i['collection']['branchname'] + if branch not in branches: + continue + + owners[branch] = {} + owners[branch]['primary'] = i['owner'] + owners[branch]['comaint'] = [] + for j in i['people']: + #if 'commit' in j['aclOrder']: + if j['aclOrder']['commit'] != None and j['username'] != owners[branch]: + owners[branch]['comaint'].append(j['username']) + + for i in sorted(branches, reverse=True): + if i in owners: + o += "%s: %s" % (i, owners[i]['primary']) + if len(owners[i]['comaint']): + o += ' - %s' % ','.join(sorted(owners[i]['comaint'])) + o += '\n' + + return (owners, o) + +def process_no_request(bug, allcomments): + '''Deal with a ticket where no request was found.''' + while 1: + os.system('clear') + print "No CVS request found in bug %d\nhttp://bugzilla.redhat.com/%d." % (bug.bug_id, bug.bug_id) + ok = raw_input('\nWhat do? (n=Next, s=Show ticket, c=Comment, q=Quit):') + if ok == 'c': + bug_comment = edit_string('') + print bug_comment + ok = raw_input("\nPost this comment to the ticket (y/n)?") + if ok == 'y': + print "Updating bugzilla..." + bug.addcomment(bug_comment) + ok = raw_input("\nClear the fedora-cvs flag (y/n)?") + if ok == 'y': + print "Clearing the flag..." + bug.updateflags({'fedora-cvs':'X', 'nomail':1}) + break + elif ok == 'n': + return True + elif ok == 'q': + return False + elif ok == 's': + print + display_bug(bug, allcomments) + return True + +def process_new_request(bug, comment, allcomments, firstfound, pkgdb, branches): + '''Parse a new package request, try to repair line wrapping, and do some + basic validity checks.''' + warned = False + warnings = [] + items = parse_prefixed_lines(comment['text']) + request = clean_request(items) + + if not firstfound: + warnings.append("WARNING: CVS request was not the last comment.") + warned = True + if not 'Package Name' in items: + warnings.append("WARNING: No package name supplied.") + warned = True + if not 'Owners' in items: + warnings.append("WARNING: No owners provided.") + warned = True + if not 'Short Description' in items: + warnings.append("WARNING: No description provided.") + warned = True + for i in request['branches']: + if i not in branches: + warnings.append("WARNING: Invalid branch %s requested" % i) + warned = True + + short_desc = bug.short_desc + m=re.search('Review Request:\s([a-zA-Z0-9_+.-]+)\s+', short_desc, re.I) + if not m: + warnings.append("WARNING: Couldn't parse package name out of bug summary.") + warned = True + elif m.group(1) != items['Package Name']: + warnings.append("WARNING: Requested package name %s doesn't match bug summary %s" % (items['Package Name'], m.group(1))) + warned = True + + req_string = new_request_string(items, bug) + bug_comment = 'GIT done (by process-git-requests).\n' + + okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' + if warned: + prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' + else: + prompt = okprompt + + # We have to loop until the user accepts the request + while 1: + # We have to loop until the user enters something that works + while 1: + os.system('clear') + if len(warnings): + print '\n'.join(warnings), "\n" + print "Currently assigned to: %s" % bug.assigned_to + print req_string + ok = raw_input(prompt) + if ok == 'a': + prompt = okprompt + warned = False + if ok == 'c': + bug_comment = edit_string('') + print bug_comment + ok = raw_input("\nPost this comment to the ticket (y/n)?") + if ok == 'y': + print "Updating bugzilla..." + bug.addcomment(bug_comment) + ok = raw_input("\nClear the fedora-cvs flag (y/n)?") + if ok == 'y': + print "Clearing the flag..." + bug.updateflags({'fedora-cvs':'X', 'nomail':1}) + return (False, True) + elif ok == 'e': + req_string = edit_string(req_string) + items=parse_prefixed_lines(req_string) + request = clean_request(items) + req_string = new_request_string(items, bug) + break + elif ok == 'n': + return (False, True) + elif ok == 'q': + return (False, False) + elif ok == 's': + print + display_bug(bug, allcomments) + elif ok == 'yes' and not warned: + bug_comment = edit_string(bug_comment) + print '\n', bug_comment + ok = raw_input('Go ahead (y/n)?') + if ok != 'y': + break + print 'Calling pkgdb...' + try: + add_package(pkgdb, request) + except Exception, e: + print "Pkgdb call failed:" + print e + raw_input('\nPress enter to continue to the next ticket.') + return (False, True) + + print 'Updating bugzilla...' + # XXX Need to handle errors here + bug.updateflags({'fedora-cvs':'+', 'nomail':1}) + bug.addcomment(bug_comment) + return (request['pkg'], True) + else: + pass + +def process_change_request(bug, comment, allcomments, firstfound, pkgdb, branches): + '''Parse a change request, try to repair line wrapping, and do some + basic validity checks.''' + owned = False + warned = False + warnings = [] + items = parse_prefixed_lines(comment['text']) + request = clean_request(items) + print "Looking up owners in pkgdb..." + (owners, owner_string) = get_pkgdb_owners(pkgdb, items['Package Name']) + + # Try to enforce EPEL branch rules + for i in owners.keys(): + if request['owner'] == owners[i]['primary'] or request['owner'] in owners[i]['comaint']: + owned = True + if not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok: + warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches.") + elif not owned and items['New Branches'].find('EL') >= 0 and owners['devel']['primary'] in epel_ok_comaint: + warnings.append("NOTE: new branch owner not owner of other branches,\n but primary devel owner is OK with EPEL branches\n as long as they comaintain.") + elif not owned: + warnings.append("WARNING: new branch owner not owner of other branches.") + warned = True + + if not firstfound: + warnings.append("WARNING: GIT request was not the last comment.") + warned = True + if not 'Package Name' in items: + warnings.append("WARNING: No package name supplied.") + warned = True + if not 'Owners' in items: + warnings.append("WARNING: No owners provided.") + warned = True + if not 'New Branches' in items: + warnings.append("WARNING: No new branches requested.") + for i in request['branches']: + if i not in branches: + warnings.append("WARNING: Invalid branch %s requested" % i) + warned = True + + short_desc = bug.short_desc + req_string = change_request_string(items, bug) + bug_comment = 'GIT done (by process-git-requests).\n' + + okprompt = 'Do it (yes=Yes, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' + if warned: + prompt = 'Warnings present!\nDo it (a=Accept warnings, n=No, e=Edit request, s=Show ticket, c=Comment, q=Quit)?' + else: + prompt = okprompt + + # We have to loop until the user accepts the request + while 1: + # We have to loop until the user enters something that works + while 1: + os.system('clear') + if len(warnings): + print '\n'.join(warnings), "\n" + print req_string + "\nCurrent branch owners - comaintainers:\n" + owner_string + ok = raw_input(prompt) + if ok == 'a': + prompt = okprompt + warned = False + if ok == 'c': + bug_comment = edit_string('') + print bug_comment + ok = raw_input("\nPost this comment to the ticket (y/n)?") + if ok == 'y': + print "Updating bugzilla..." + bug.addcomment(bug_comment) + ok = raw_input("\nClear the fedora-cvs flag (y/n)?") + if ok == 'y': + print "Clearing the flag..." + bug.updateflags({'fedora-cvs':'X', 'nomail':1}) + return (False, True) + elif ok == 'e': + req_string = edit_string(req_string) + items=parse_prefixed_lines(req_string) + request = clean_request(items) + req_string = change_request_string(items, bug) + break + elif ok == 'n': + return (False, True) + elif ok == 'q': + return (False, False) + elif ok == 's': + print + display_bug(bug, allcomments) + elif ok == 'yes' and not warned: + bug_comment = edit_string(bug_comment) + print '\n', bug_comment + ok = raw_input('Go ahead (y/n)?') + if ok != 'y': + break + print 'Calling pkgdb...' + try: + edit_package(pkgdb, request) + except Exception, e: + print "Pkgdb call failed:" + print e + raw_input('\nPress enter to continue to the next ticket.') + return (False, True) + + print 'Updating bugzilla...' + # XXX Need to handle errors here + bug.updateflags({'fedora-cvs':'+', 'nomail':1}) + bug.addcomment(bug_comment) + return (request['pkg'], True) + else: + pass + +if __name__ == '__main__': + branches = {} + processed = [] + options = parse_commandline() + print "Connecting to bugzilla..." + bz = bugzilla.Bugzilla(url=options.url) + print "Querying bugzilla..." + (bugs, comments) = run_query(bz) + print "Done; got %d." % len(bugs) + if not len(bugs): + print "No requests to process!" + exit(0) + + print "Connecting to pkgdb..." + config = parse_pkgdb_config() + pkgdb = PackageDB(config['pkgdb.url'], username=options.user, + debug=options.debug) + print "Getting valid branches...." + for i in pkgdb.get_collection_list(eol=False): + branches[i[0]['branchname']] = 1 + print "Done." + print + + # Iterate over bugs + for i in bugs: + firstfound = True + type = '' + print "Parsing bug %d - https://bugzilla.redhat.com/%d" % (i.bug_id, i.bug_id) + for j in reversed(comments['bugs'][str(i.bug_id)]['comments']): + if 'New Package CVS Request' in j['text']: + type = 'new' + break + if 'Package Change Request' in j['text']: + type = 'change' + break + firstfound = False + else: + if not process_no_request(i, comments['bugs'][str(i.bug_id)]['comments']): + break + + if type == 'new': + (package, more) = process_new_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches) + if package: + processed.append(package) + if not more: + break + elif type == 'change': + (package, more) = process_change_request(i, j, comments['bugs'][str(i.bug_id)]['comments'], firstfound, pkgdb, branches) + if package: + processed.append(package) + if not more: + break + + if len(processed): + print '\nYou must now run this on the git server\nto set up the git repository:' + print '/usr/local/bin/pkgdb2branch.py ' + ' '.join(processed) + + sys.exit(0) diff --git a/roles/distgit/files/redirect.conf b/roles/distgit/files/redirect.conf new file mode 100644 index 0000000000..dc1b6a1133 --- /dev/null +++ b/roles/distgit/files/redirect.conf @@ -0,0 +1,160 @@ +RedirectMatch permanent ^/$ http://pkgs.fedoraproject.org/cgit/ + +RewriteEngine on + +# rewrite log +# set level to 0 to turn off and speed things up +# when debugging - set to 3 and you can see what the hell is going on +RewriteLogLevel 0 +RewriteLog "/var/log/httpd/git-rewrite.log" + +# Use cgit and redirect (some) old gitweb-caching things +RewriteRule ^/cgit-data/(.*)$ /cgit-data/$1 [L,PT] +RewriteRule ^/cgit/(.*)$ /cgit/$1 [L,PT] + +# blob +RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob/(.+)/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=blob;h=(.+);hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%5?id=%3;id2=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=blob;hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%4?id=%3 [R,L,NE] + +RewriteCond %{query_string} p=(.+)(\.git);a=blob;f=(.+);h=(.+);hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%3?id=%4;id2=%5 [R,L,NE] + +RewriteCond %{query_string} p=(.+)(\.git);a=blob;f=(.+);h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%3?id=%4 [R,L,NE] + +# tree +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+)/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tree;h=(.+);hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%5?id=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tree;hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/%4?id=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+)/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tree;h=(.+);hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/?id=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tree;hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/?id=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tree [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tree +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tree/? [R,L,NE] + +# commitdiff +RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+)/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=blobdiff;h=(.+);hp=(.+);hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/%6?id2=%4;id=%3;id3=%5 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+)/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=commitdiff;h=(.+);hp=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/?id=%4;id2=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/commitdiff/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=commitdiff;h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/diff/?id=%3 [R,L,NE] + +# commit +RewriteCond %{REQUEST_URI} /(.+)(\.git)/commit/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=commit;h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/commit/?id=%3 [R,L,NE] + +# summary +RewriteCond %{REQUEST_URI} /(.+)(\.git)/summary [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=summary +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE] + +# shortlog +RewriteCond %{REQUEST_URI} /(.+)(\.git)/shortlog/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=shortlog;h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/shortlog [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=shortlog +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/? [R,L,NE] + +# log +RewriteCond %{REQUEST_URI} /(.+)(\.git)/log/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=log;h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/log [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=log +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log? [R,L,NE] + +# history +RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+)/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=history;h=(.+);hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%5?id=%4 [R,L,NE] + +RewriteCond %{query_string} p=(.+)(\.git);a=history;f=(.+);h=(.+);hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%3?id=%4;id2=%5 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+):/(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%4?id=%3 [R,L,NE] + +RewriteCond %{query_string} p=(.+)(\.git);a=history;f=(.+);h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%3?id=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+)/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=history;h=(.+);hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=history;hb=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/%4?id=%3 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/history/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=history;hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/log/?id=%3 [R,L,NE] + +# tag +RewriteCond %{REQUEST_URI} /(.+)(\.git)/tag/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=tag;h=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/tag/?id=%3 [R,L,NE] + +# blob_plain +RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob_plain/(.+):/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;h=(.+);f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%4?id=%3 [R,L,NE] + +RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;f=(.+);hb=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%3?id2=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/blob_plain/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=blob_plain;f=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/plain/%3 [R,L,NE] + +# rss|atom +RewriteCond %{REQUEST_URI} /(.+)(\.git)/(rss|atom)/refs/heads/(.+) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=(rss|atom);h=refsheads/(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/atom?h=%4 [R,L,NE] + +RewriteCond %{REQUEST_URI} /(.+)(\.git)/(rss|atom) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=(rss|atom) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/atom? [R,L,NE] + +# snapshot +RewriteCond %{REQUEST_URI} /(.+)(\.git)/snapshot/(.+)(\.tar\.gz|\.tar\.bz2) [OR] +RewriteCond %{query_string} p=(.+)(\.git);a=snapshot;h=(.+);sf=(.+) +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/snapshot/%3.tar.gz [R,L,NE] + +# base old gitweb project +RewriteCond %{REQUEST_URI} /gitweb/(.+)\.git.* [OR] +RewriteCond %{query_string} p=(.+)\.git.* +RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE] + +# Fail safes incase nothing above matches, try at least to put the person in the project +#RewriteCond %{REQUEST_URI} /(.+)\.git.* [OR] +#RewriteCond %{query_string} p=(.+)\.git.* +#RewriteRule ^/.*$ http://pkgs.fedoraproject.org/cgit/%1.git/? [R,L,NE] + +# Or else in the root of cgit +#RewriteRule ^.* http://pkgs.fedoraproject.org/cgit/ [R,L,NE] diff --git a/roles/distgit/files/setup_git_package b/roles/distgit/files/setup_git_package new file mode 100644 index 0000000000..73f5d28bc5 --- /dev/null +++ b/roles/distgit/files/setup_git_package @@ -0,0 +1,125 @@ +#!/bin/bash +# +# Create a new repo. +# THIS HAS TO BE RUN ON THE GIT SERVER! + +# WARNING: +# This file is maintained within ansible +# All local changes will be lost. + + +# Figure out the environment we're running in +GITROOT=/srv/git/rpms + +# check if a moron is driving me +if [ ! -d $GITROOT ] ; then + # we're not on the git server (this check is fragile) + echo "ERROR: This script has to be run on the git server." + echo "ERROR: Homer sez 'Duh'." + exit -9 +fi + +# Local variables +VERBOSE=0 +TEST= +IGNORE= +AUTHOR="Fedora Release Engineering " +GIT_SSH_URL="ssh://localhost" + +Usage() { + cat < + + Creates a new repo for + +Options: + -h,--help This help message +EOF +} + +if [ $# -gt 2 ]; then + Usage + exit -1 +fi + +# parse the arguments +while [ -n "$1" ] ; do + case "$1" in + -h | --help ) + Usage + exit 0 + ;; + + * ) + PACKAGE="$1" + ;; + esac + shift +done + +# I hate shell scripting. I'm sure the above is totally wrong + +# check the arguments +if [ -z "$PACKAGE" ] ; then + Usage + exit -1 +fi + +# Sanity checks before we start doing damage +[ $VERBOSE -gt 1 ] && echo "Checking package $PACKAGE..." +if [ -d $GITROOT/$PACKAGE.git ] ; then + echo "ERROR: Package module $PACKAGE already exists!" >&2 + exit -1 +fi + +# Just don't run as root, mmkey? +if [ "$(id -un)" = "root" ] ; then + echo "Please run this script as yourself" + exit -3 +fi + +# "global" permissions check +if [ ! -w $GITROOT ] ; then + echo "ERROR: You can not write to $GITROOT" + echo "ERROR: You can not create repos" + exit -1 +fi + +# Now start working on creating those branches +# Create a tmpdir to do some git work in +TMPDIR=$(mktemp -d /tmp/tmpXXXXXX) + +# First create the master repo +mkdir $GITROOT/$PACKAGE.git +pushd $GITROOT/$PACKAGE.git >/dev/null +git init -q --shared --bare +echo "$PACKAGE" > description # This is used to figure out who to send mail to. +git config --add hooks.mailinglist "$PACKAGE-owner@fedoraproject.org,scm-commits@lists.fedoraproject.org" +git config --add hooks.maildomain fedoraproject.org +popd >/dev/null + +# Now clone that repo and create the .gitignore and sources file +git clone -q /srv/git/rpms/$PACKAGE.git $TMPDIR/$PACKAGE +pushd $TMPDIR/$PACKAGE >/dev/null +touch .gitignore sources +git add . +git commit -q -m 'Initial setup of the repo' --author "$AUTHOR" +git push -q origin master +popd >/dev/null + +# Put our special update hooks in place +ln -s /usr/share/gitolite/hooks/common/update $GITROOT/$PACKAGE.git/hooks/ + +mkdir -p $GITROOT/$PACKAGE.git/hooks/post-receive-chained.d +ln -s /usr/share/git-core/mail-hooks/gnome-post-receive-email \ + $GITROOT/$PACKAGE.git/hooks/post-receive-chained.d/post-receive-email +ln -s /usr/share/git-core/post-receive-fedmsg \ + $GITROOT/$PACKAGE.git/hooks/post-receive-chained.d/post-receive-fedmsg + +# This one kicks off all the others in post-receive-chained.d +ln -s /usr/share/git-core/post-receive-chained \ + $GITROOT/$PACKAGE.git/hooks/post-receive + +rm -rf $TMPDIR +echo "Done." diff --git a/roles/distgit/files/ssl.conf b/roles/distgit/files/ssl.conf new file mode 100644 index 0000000000..e42dd66b2c --- /dev/null +++ b/roles/distgit/files/ssl.conf @@ -0,0 +1,16 @@ +# +# This is the Apache server configuration file providing SSL support. +# It contains the configuration directives to instruct the server how to +# serve pages over an https connection. For detailing information about these +# directives see +# +# For the moment, see for this info. +# The documents are still being prepared from material donated by the +# modssl project. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# + +LoadModule ssl_module modules/mod_ssl.so diff --git a/roles/distgit/files/updatecrl.sh b/roles/distgit/files/updatecrl.sh new file mode 100644 index 0000000000..c7649b3b9d --- /dev/null +++ b/roles/distgit/files/updatecrl.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +URL=https://admin.fedoraproject.org/ca/crl.pem +OLD=/etc/pki/tls/crl.pem +NEW=/tmp/crl.pem + +wget $URL -O $NEW +OLDUPDATE=`openssl crl -in $OLD -noout -lastupdate` +NEWUPDATE=`openssl crl -in $NEW -noout -lastupdate` + +if [ "$OLDUPDATE" != "$NEWUPDATE" ]; then + mv $NEW $OLD + restorecon $OLD + /etc/init.d/httpd graceful + echo "updated to $NEWUPDATE" +fi diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml new file mode 100644 index 0000000000..336439f7b6 --- /dev/null +++ b/roles/distgit/tasks/main.yml @@ -0,0 +1,147 @@ +--- +# tasklist for setting up Dist Git +# +# This is a bit complex, so I'm dividing it into sections. + +# -- Common ---------------------------------------------- +# This is very basic stuff that is needed by multiple of the next sections. +- name: install the needed packages + yum: pkg={{item}} state=present + with_items: + - git + - httpd + - mod_ssl + - python-fedmsg-genacls + +- name: set some sysctl options + sysctl: name=vm.zone_reclaim_mode value=1 state=present + when: env != "staging" + +- name: create the packager group + group: name=packager gid=415 state=present + +- name: install the httpd config directory + copy: src=pkgs.fedoraproject.org.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf + file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org state=directory + notify: + - restart httpd + +- name: install the mod_ssl configuration + copy: src=ssl.conf dest=/etc/httpd/conf.d/ssl.conf + notify: + - restart httpd + +- name: allow httpd to access the files on NFS + seboolean: name=httpd_use_nfs state=yes persistent=yes + +# -- Dist Git -------------------------------------------- +# This is the Git setup itself: group, root directory, scripts,... +# +# Requires: roles/git/hooks +# Requires: roles/git/make_checkout_seed +# Requires: roles/git/server +- name: create the distgit root directory) + file: dest=/srv/git state=directory mode=0755 + file: dest=/srv/git/rpms state=directory mode=2775 group=packager + +- name: install the distgit scripts + copy: src={{item}} dest=/usr/local/bin/{{item}} owner=root group=root mode=0755 + with_items: + - setup_git_package + - mkbranch + - pkgdb2-clone + - pkgdb2branch.py + - process-git-requests + +- name: install the Dist Git-related httpd config + copy: src=git-smart-http.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/git-smart-http.conf + notify: + - restart httpd + + +# -- Gitolite -------------------------------------------- +# This is the permission management for package maintainers, using Gitolite. +# +# Requires: roles/fedmsg/base +# Requires: roles/fedmsg/hub +# Requires: roles/gitolite/base +# Requires: roles/gitolite/check_fedmsg_hooks +- name: mount the lookaside path + mount: > + src=vtap-fedora-nfs01.storage.phx2.redhat.com:/vol/fedora_sourcecache + name=/srv/cache/lookaside + fstype=nfs + opts=rw,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3 + state=mounted + when: env != "staging" + +- name: create the /var/log/gitolite directory + file: path=/var/log/gitolite owner=root group=packager state=directory mode=2775 + +- name: create the gen-acls user + group: name=gen-acls gid=417 state=present + user: name=gen-acls comment="dummy system account for the gen-acls fedmsg job" uid=417 group=gen-acls shell=/bin/bash home=/ + +- name: create the /etc/gitolite/conf directory + file: path=/etc/gitolite/conf owner=gen-acls group=gen-acls state=directory mode=0755 + +- name: create /etc/gitolite/gitolite.rc + copy: src=gitolite.rc dest=/etc/gitolite/gitolite.rc owner=root group=root mode=0755 + +- name: install the gitolite scripts + copy: src={{item}} dest=/usr/local/bin/{{item}} mode=0755 + with_items: + - genacls.pkgdb + - genacls.sh + +- name: install the fedmsg configuration + copy: src=fedmsg-genacls-config.py dest=/etc/fedmsg.d/genacls.py owner=root group=root mode=0644 + + +# -- CGit ------------------------------------------------ +# This is the pretty web view of the repositories, using CGit. +# +# Requires: roles/cgit/base +# Requires: roles/cgit/clean_lock_cron +# Requires: roles/cgit/make_pkgs_list +- name: install the cgitrc file + copy: src=cgitrc dest=/etc/cgitrc + +- name: install the CGit-related httpd config + copy: src=redirect.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/redirect.conf + notify: + - restart httpd + + +# -- Lookaside Cache ------------------------------------- +# This is the annex to Dist Git, where we host source tarballs. +# +# Requires: clamav +- name: install the Lookaside Cache httpd configs + copy: src={{item}} dest=/etc/httpd/conf.d/pkgs.fedoraproject.org/{{item}} + with_items: + - lookaside.conf + - lookaside-upload.conf + notify: + - restart httpd + +- name: create the Lookaside Cache root directory + file: dest=/srv/cache/lookaside/pkgs state=directory + +- name: install the certificates + copy: src={{private}}/fedora-ca.cert dest=/etc/httpd/conf/cacert.pem + copy: src={{private}}/pkgs.fedoraproject.org_key_and_cert.pem dest=/etc/httpd/conf/pkgs.fedoraproject.org_key_and_cert.pem owner=apache mode=0400 + +- name: install, run, and schedule the updatecrl.sh script + copy: src=updatecrl.sh dest=/usr/local/bin/updatecrl.sh owner=root mode=755 + command: /usr/local/bin/updatecrl.sh creates=/etc/pki/tls/crl.pem + cron: > + name="updatecrl" cron_file="ansible-updatecrl" + minute=0 + user=root + job="/usr/local/bin/updatecrl.sh" + +- name: install the upload CGI script + copy: src=dist-git-upload.cgi dest=/srv/web/upload.cgi owner=root group=root mode=0755 + notify: + - restart httpd From a03781965d5625230d1d309d29859ff3c6f35017 Mon Sep 17 00:00:00 2001 From: Mathieu Bridon Date: Thu, 17 Jul 2014 18:35:49 +0200 Subject: [PATCH 127/193] Setup the production and staging Dist Git --- inventory/group_vars/pkgs | 45 +++++++ .../host_vars/pkgs01.phx2.fedoraproject.org | 3 + .../pkgs01.stg.phx2.fedoraproject.org | 3 + playbooks/groups/pkgs.yml | 56 ++++++++ roles/base/files/ssh/sshd_config.pkgs | 121 ++++++++++++++++++ 5 files changed, 228 insertions(+) create mode 100644 inventory/group_vars/pkgs create mode 100644 inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org create mode 100644 playbooks/groups/pkgs.yml create mode 100644 roles/base/files/ssh/sshd_config.pkgs diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs new file mode 100644 index 0000000000..625f87b904 --- /dev/null +++ b/inventory/group_vars/pkgs @@ -0,0 +1,45 @@ +--- +# TODO: Define resources for this group of hosts here? + +tcp_ports: [80, 443, 9418, + # These 16 ports are used by fedmsg. One for each wsgi thread. + 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, + 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] + +fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_restricted_app: /usr/bin/gl-auth-command +fas_client_admin_app: /usr/bin/gl-auth-command -s +fas_client_ssh_groups = @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc + +git_group: packager +git_port: 9418 +git_server: /usr/libexec/git-core/git-daemon +git_server_args: --export-all --syslog --inetd --verbose +git_basepath: /srv/git/rpms + +clamscan_mailto: admin@fedoraproject.org +clamscan_paths: +- /srv/cache/lookaside/pkgs +clamscan_excludes: +- clamav- +- amavisd-new-2.3.3.tar.gz +- bro-20080804.tgz +- mailman- +- sagator- +- nicotine +- fwsnort-1.0.6.tar.gz +- psad-2.1.7.tar.bz2 +- pymilter- +- linkchecker- + +# These are consumed by a task in roles/fedmsg/base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: sysadmin +- service:scm + owner: root + group: packager +- service: lookaside + owner: root + group: apache diff --git a/inventory/host_vars/pkgs01.phx2.fedoraproject.org b/inventory/host_vars/pkgs01.phx2.fedoraproject.org index fbc0826155..be260f621c 100644 --- a/inventory/host_vars/pkgs01.phx2.fedoraproject.org +++ b/inventory/host_vars/pkgs01.phx2.fedoraproject.org @@ -1,2 +1,5 @@ --- host_backup_targets: ['/srv'] + +nm: 255.255.255.0 +eth1_ip: 10.5.127.67 diff --git a/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org b/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..ea61164474 --- /dev/null +++ b/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org @@ -0,0 +1,3 @@ +--- +nm: 255.255.255.0 +eth0_ip: 10.5.126.83 diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml new file mode 100644 index 0000000000..35bf7f6eeb --- /dev/null +++ b/playbooks/groups/pkgs.yml @@ -0,0 +1,56 @@ +- name: make pkgs + hosts: pkgs:pkgs-stg + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: pkgs:pkgs-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - denyhosts + - nagios_client + - fas_client + - collectd/base + - fedmsg/base + - fedmsg/hub + - sudo + - git/hooks + - git/make_checkout_seed + - git/server + - gitolite/base + - gitolite/check_fedmsg_hooks + - cgit/base + - cgit/clean_lock_cron + - cgit/make_pkgs_list + - clamav + - distgit + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/apache.yml" + - include: "{{ tasks }}/drbackupkey.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/base/files/ssh/sshd_config.pkgs b/roles/base/files/ssh/sshd_config.pkgs new file mode 100644 index 0000000000..7fddcd6ffd --- /dev/null +++ b/roles/base/files/ssh/sshd_config.pkgs @@ -0,0 +1,121 @@ +# $OpenBSD: sshd_config,v 1.73 2005/12/06 22:38:28 reyk Exp $ + +# This is the sshd server system-wide configuration file. See +# sshd_config(5) for more information. + +# This sshd was compiled with PATH=/usr/local/bin:/bin:/usr/bin + +# The strategy used for options in the default sshd_config shipped with +# OpenSSH is to specify options with their default value where +# possible, but leave them commented. Uncommented options change a +# default value. + +#Port 22 +#Protocol 2,1 +Protocol 2 +#AddressFamily any +#ListenAddress 0.0.0.0 +#ListenAddress :: + +# HostKey for protocol version 1 +#HostKey /etc/ssh/ssh_host_key +# HostKeys for protocol version 2 +HostKey /etc/ssh/ssh_host_rsa_key +#HostKey /etc/ssh/ssh_host_dsa_key + +# Lifetime and size of ephemeral version 1 server key +#KeyRegenerationInterval 1h +#ServerKeyBits 768 + +# Logging +# obsoletes QuietMode and FascistLogging +#SyslogFacility AUTH +SyslogFacility AUTHPRIV +LogLevel VERBOSE + +# Authentication: + +#LoginGraceTime 2m +PermitRootLogin without-password +StrictModes yes +#MaxAuthTries 6 + +#RSAAuthentication yes +#PubkeyAuthentication yes +#AuthorizedKeysFile .ssh/authorized_keys + +# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts +#RhostsRSAAuthentication no +# similar for protocol version 2 +#HostbasedAuthentication no +# Change to yes if you don't trust ~/.ssh/known_hosts for +# RhostsRSAAuthentication and HostbasedAuthentication +#IgnoreUserKnownHosts no +# Don't read the user's ~/.rhosts and ~/.shosts files +#IgnoreRhosts yes + +# To disable tunneled clear text passwords, change to no here! +#PasswordAuthentication yes +#PermitEmptyPasswords no +PasswordAuthentication no + +# Change to no to disable s/key passwords +#ChallengeResponseAuthentication yes +ChallengeResponseAuthentication no + +# Kerberos options +#KerberosAuthentication no +#KerberosOrLocalPasswd yes +#KerberosTicketCleanup yes +#KerberosGetAFSToken no + +# GSSAPI options +#GSSAPIAuthentication no +GSSAPIAuthentication no +#GSSAPICleanupCredentials yes +GSSAPICleanupCredentials no + +# Set this to 'yes' to enable PAM authentication, account processing, +# and session processing. If this is enabled, PAM authentication will +# be allowed through the ChallengeResponseAuthentication mechanism. +# Depending on your PAM configuration, this may bypass the setting of +# PasswordAuthentication, PermitEmptyPasswords, and +# "PermitRootLogin without-password". If you just want the PAM account and +# session checks to run without PAM authentication, then enable this but set +# ChallengeResponseAuthentication=no +#UsePAM no +UsePAM yes + +# Accept locale-related environment variables +AcceptEnv LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES +AcceptEnv LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT +AcceptEnv LC_IDENTIFICATION LC_ALL +#AllowTcpForwarding yes +AllowTcpForwarding no + + +#GatewayPorts no +#X11Forwarding no +X11Forwarding yes +#X11DisplayOffset 10 +#X11UseLocalhost yes +#PrintMotd yes +#PrintLastLog yes +#TCPKeepAlive yes +#UseLogin no +#UsePrivilegeSeparation yes +#PermitUserEnvironment no +#Compression delayed +#ClientAliveInterval 0 +#ClientAliveCountMax 3 +#ShowPatchLevel no +#UseDNS yes +#PidFile /var/run/sshd.pid +#MaxStartups 10 +PermitTunnel no + +# no default banner path +#Banner /some/path + +# override default of no subsystems +Subsystem sftp /usr/libexec/openssh/sftp-server From e6215d8c18098199d0143c458d1af5f7f89b69cf Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:37:16 +0000 Subject: [PATCH 128/193] Don't Enable on prod yet --- playbooks/groups/pkgs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index 35bf7f6eeb..ff03341c79 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -1,5 +1,5 @@ - name: make pkgs - hosts: pkgs:pkgs-stg + hosts: pkgs-stg user: root gather_facts: False @@ -15,7 +15,7 @@ - include: "{{ handlers }}/restart_services.yml" - name: make the box be real - hosts: pkgs:pkgs-stg + hosts: pkgs-stg user: root gather_facts: True accelerate: "{{ accelerated }}" From 4be1402087fd7c798cc5022c01e81a581d421a97 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:48:55 +0000 Subject: [PATCH 129/193] Confim that ansible doesn't like @cvs --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 625f87b904..9515cb81cb 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -9,7 +9,7 @@ tcp_ports: [80, 443, 9418, fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc fas_client_restricted_app: /usr/bin/gl-auth-command fas_client_admin_app: /usr/bin/gl-auth-command -s -fas_client_ssh_groups = @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_ssh_groups = sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc git_group: packager git_port: 9418 From 4acd584b4d31cde3a548e7d638f333d00d5b8436 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:49:40 +0000 Subject: [PATCH 130/193] Revert "Confim that ansible doesn't like @cvs" This reverts commit 4be1402087fd7c798cc5022c01e81a581d421a97. --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 9515cb81cb..625f87b904 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -9,7 +9,7 @@ tcp_ports: [80, 443, 9418, fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc fas_client_restricted_app: /usr/bin/gl-auth-command fas_client_admin_app: /usr/bin/gl-auth-command -s -fas_client_ssh_groups = sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_ssh_groups = @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc git_group: packager git_port: 9418 From 0f97b1235236fa3856da5e2f2438aa5dfc9cb6ee Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:56:56 +0000 Subject: [PATCH 131/193] Quote this variable. --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 625f87b904..8ab61f8617 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -9,7 +9,7 @@ tcp_ports: [80, 443, 9418, fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc fas_client_restricted_app: /usr/bin/gl-auth-command fas_client_admin_app: /usr/bin/gl-auth-command -s -fas_client_ssh_groups = @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_ssh_groups = "@cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc" git_group: packager git_port: 9418 From 1021610b37e8b4d1868e4a35e382c87600e1ba73 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:57:36 +0000 Subject: [PATCH 132/193] Bad equals. --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 8ab61f8617..b2a8bad046 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -9,7 +9,7 @@ tcp_ports: [80, 443, 9418, fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc fas_client_restricted_app: /usr/bin/gl-auth-command fas_client_admin_app: /usr/bin/gl-auth-command -s -fas_client_ssh_groups = "@cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc" +fas_client_ssh_groups: @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc git_group: packager git_port: 9418 From 4c504f65c24764c8265677164a94a298a1008909 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:58:06 +0000 Subject: [PATCH 133/193] But we still need to quote the @ --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index b2a8bad046..170a411d74 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -9,7 +9,7 @@ tcp_ports: [80, 443, 9418, fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc fas_client_restricted_app: /usr/bin/gl-auth-command fas_client_admin_app: /usr/bin/gl-auth-command -s -fas_client_ssh_groups: @cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_ssh_groups: "@cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc" git_group: packager git_port: 9418 From f87ee34580bca2748559fe6bd445b905650fe2b4 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 18:58:41 +0000 Subject: [PATCH 134/193] Add missing space --- inventory/group_vars/pkgs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index 170a411d74..c5789f3ba2 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -37,7 +37,7 @@ fedmsg_certs: - service: shell owner: root group: sysadmin -- service:scm +- service: scm owner: root group: packager - service: lookaside From 1109cae0121fab257b9c99799e85fe85be33d0c3 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:00:19 +0000 Subject: [PATCH 135/193] Fix task that was trying to do two things at once. --- roles/cgit/make_pkgs_list/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/cgit/make_pkgs_list/tasks/main.yml b/roles/cgit/make_pkgs_list/tasks/main.yml index fcf1032f90..972905e857 100644 --- a/roles/cgit/make_pkgs_list/tasks/main.yml +++ b/roles/cgit/make_pkgs_list/tasks/main.yml @@ -3,6 +3,8 @@ - name: install the script and schedule its execution copy: src=make-cgit-pkgs-list.sh dest=/usr/local/bin/make-cgit-pkgs-list.sh mode=0755 + +- name: install the cron job cron: > name="make-cgit-pkgs-list" cron_file="ansible-make-cgit-pkgs-list" minute=*/10 From ac645625642e334ee65e96781e1c6e233998a960 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:02:03 +0000 Subject: [PATCH 136/193] Fix another task with 2 items in it. --- roles/git/make_checkout_seed/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/git/make_checkout_seed/tasks/main.yml b/roles/git/make_checkout_seed/tasks/main.yml index 86a35bdeb9..747d5f48d2 100644 --- a/roles/git/make_checkout_seed/tasks/main.yml +++ b/roles/git/make_checkout_seed/tasks/main.yml @@ -6,6 +6,8 @@ - name: install the script and schedule its execution copy: src=make-git-checkout-seed.sh dest=/usr/local/bin/make-git-checkout-seed.sh mode=0755 + +- name: install cron job. cron: > name="make-git-checkout-seed" cron_file="ansible-make-git-checkout-seed" minute=0 hour=2 From b4996ad8f1cc89881aaef15c907f41df9e9c5fc4 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:03:54 +0000 Subject: [PATCH 137/193] Fix another 2 task item --- roles/gitolite/check_fedmsg_hooks/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/gitolite/check_fedmsg_hooks/tasks/main.yml b/roles/gitolite/check_fedmsg_hooks/tasks/main.yml index 0d25329fdf..3dcae95441 100644 --- a/roles/gitolite/check_fedmsg_hooks/tasks/main.yml +++ b/roles/gitolite/check_fedmsg_hooks/tasks/main.yml @@ -4,10 +4,12 @@ - name: install the needed packages yum: pkg=git state=present -- name: install and schedule the script +- name: install the script copy: > src=check-perms.py dest=/usr/local/bin/git-check-perms owner=root group=root mode=0755 + +- name: install the cron job for the script cron: > name="git-check-perms" cron_file="ansible-git-check-perms" minute=10 hour=0 weekday=3 From c9e14c8e0be598ff25601a31cf649114763c5a00 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:05:26 +0000 Subject: [PATCH 138/193] Fix multiple tasks in play --- roles/cgit/base/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/cgit/base/tasks/main.yml b/roles/cgit/base/tasks/main.yml index abb35894eb..3e7b810fe4 100644 --- a/roles/cgit/base/tasks/main.yml +++ b/roles/cgit/base/tasks/main.yml @@ -8,9 +8,13 @@ - lua-md5 # See https://github.com/robyduck/cgit-custom -- name: add our custom theme +- name: Make sure cgit directory exists file: dest=/usr/share/cgit state=directory owner=root group=root mode=0755 + +- name: Make sure cgit images directory exists file: dest=/usr/share/cgit/images state=directory owner=root group=root mode=0755 + +- name: Copy cgit them into place. copy: src={{item}} dest=/usr/share/cgit/{{item}} owner=root group=root mode=0644 with_items: - cgit.css From 21ce3334640ade15e754ca2cdc460a9e01ce9b2f Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:06:29 +0000 Subject: [PATCH 139/193] Fix another multiple tasks issue --- roles/clamav/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml index 335a2f76d2..0bf8bab8f8 100644 --- a/roles/clamav/tasks/main.yml +++ b/roles/clamav/tasks/main.yml @@ -26,11 +26,13 @@ - name: setup the freshclam cron job copy: src=freshclam-cron dest=/etc/cron.daily/freshclam -- name: setup the periodic clam scan +- name: setup the periodic clam scan script template: > src=clamscan.sh.j2 dest=/usr/local/bin/clamscan.sh mode=0755 + +- name: setup cron job for clam scan cron: > name="clamscan" minute={{ clamscan_minute | default(35) }} From c154fee038cb69ca5ffef1c574aa330e13b998fa Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:08:54 +0000 Subject: [PATCH 140/193] Split another task --- roles/distgit/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 336439f7b6..b49e53c0cf 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -20,8 +20,12 @@ - name: create the packager group group: name=packager gid=415 state=present -- name: install the httpd config directory +- name: install the httpd config file copy: src=pkgs.fedoraproject.org.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf + notify: + - restart httpd + +- name: install the httpd config directory file: dest=/etc/httpd/conf.d/pkgs.fedoraproject.org state=directory notify: - restart httpd From 9c65d0e6736a9f1139cccef772904940feef4838 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:10:41 +0000 Subject: [PATCH 141/193] Split another. --- roles/distgit/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index b49e53c0cf..10bf5eddb1 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -82,8 +82,10 @@ - name: create the /var/log/gitolite directory file: path=/var/log/gitolite owner=root group=packager state=directory mode=2775 -- name: create the gen-acls user +- name: create the gen-acls group group: name=gen-acls gid=417 state=present + +- name: create the gen-acls user user: name=gen-acls comment="dummy system account for the gen-acls fedmsg job" uid=417 group=gen-acls shell=/bin/bash home=/ - name: create the /etc/gitolite/conf directory From 9bd6f9cfc14aaaa5b8d49e0d23bcc6db04792f50 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:12:00 +0000 Subject: [PATCH 142/193] Split this task into 3 --- roles/distgit/tasks/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 10bf5eddb1..240c5c0e03 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -138,9 +138,13 @@ copy: src={{private}}/fedora-ca.cert dest=/etc/httpd/conf/cacert.pem copy: src={{private}}/pkgs.fedoraproject.org_key_and_cert.pem dest=/etc/httpd/conf/pkgs.fedoraproject.org_key_and_cert.pem owner=apache mode=0400 -- name: install, run, and schedule the updatecrl.sh script +- name: install the updatecrl.sh script copy: src=updatecrl.sh dest=/usr/local/bin/updatecrl.sh owner=root mode=755 + +- name: run the updatecrl script command: /usr/local/bin/updatecrl.sh creates=/etc/pki/tls/crl.pem + +- name: schedule cron job to run the updatectl script cron: > name="updatecrl" cron_file="ansible-updatecrl" minute=0 From 1c7a44fe4499970a5bb859d3fb83589536a3006e Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:12:53 +0000 Subject: [PATCH 143/193] Split this one out into 2 tasks --- tasks/drbackupkey.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml index 166230c580..fdc1c0f86c 100644 --- a/tasks/drbackupkey.yml +++ b/tasks/drbackupkey.yml @@ -1,6 +1,8 @@ --- -- name: ensure the user exists +- name: ensure the drbackup group exists group: name=drbackup state=present + +- name: ensure the drbackup user exists user: name=drbackup comment="DR Backup User" group=drbackup shell=/bin/bash home=/var/lib/drbackup file: dest=/var/lib/drbackup/ state=directory owner=drbackup group=drbackup mode=0700 From 2fc6902e4ef70d38676bb6181bdb4bba84cda8ee Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:13:49 +0000 Subject: [PATCH 144/193] Missed these two --- tasks/drbackupkey.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml index fdc1c0f86c..6f7b214164 100644 --- a/tasks/drbackupkey.yml +++ b/tasks/drbackupkey.yml @@ -4,10 +4,14 @@ - name: ensure the drbackup user exists user: name=drbackup comment="DR Backup User" group=drbackup shell=/bin/bash home=/var/lib/drbackup + +- name: Make sure the drbackup homedir exists file: dest=/var/lib/drbackup/ state=directory owner=drbackup group=drbackup mode=0700 - name: install the authorized SSH key file: dest=/var/lib/drbackup/.ssh/ state=directory owner=drbackup group=drbackup mode=0700 + +- name: install the backup ssh keys copy: src={{private}}/backup.pub dest=/var/lib/drbackup/.ssh/authorized_keys owner=drbackup group=drbackup mode=0600 - name: deploy the confine-ssh script From 8de36642709ebbbb9660029c9bcbce960fd7e468 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:14:46 +0000 Subject: [PATCH 145/193] tasks isn't valid here. --- tasks/drbackupkey.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml index 6f7b214164..f4443bd7a4 100644 --- a/tasks/drbackupkey.yml +++ b/tasks/drbackupkey.yml @@ -14,6 +14,7 @@ - name: install the backup ssh keys copy: src={{private}}/backup.pub dest=/var/lib/drbackup/.ssh/authorized_keys owner=drbackup group=drbackup mode=0600 -- name: deploy the confine-ssh script - tasks: - - include: "{{tasks}}/confine-ssh.yml" +# +# deploy the confine-ssh script +# +- include: "{{tasks}}/confine-ssh.yml" From cb8dd8a1c0edbd5a06ab68ffc3efccf756e5790b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:15:19 +0000 Subject: [PATCH 146/193] File has a _ in it not a - --- tasks/drbackupkey.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml index f4443bd7a4..0f2741fd80 100644 --- a/tasks/drbackupkey.yml +++ b/tasks/drbackupkey.yml @@ -17,4 +17,4 @@ # # deploy the confine-ssh script # -- include: "{{tasks}}/confine-ssh.yml" +- include: "{{tasks}}/confine_ssh.yml" From 176144ab95b07ae91941b724e1ffe1af6f8eef9b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:18:28 +0000 Subject: [PATCH 147/193] Add more info on pkgs01.stg --- inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org b/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org index ea61164474..6067fc04ed 100644 --- a/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org +++ b/inventory/host_vars/pkgs01.stg.phx2.fedoraproject.org @@ -1,3 +1,10 @@ --- -nm: 255.255.255.0 eth0_ip: 10.5.126.83 +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-7 +ks_repo: http://10.5.126.23/repo/rhel/RHEL7-x86_64/ +volgroup: /dev/vg_virthost16 +vmhost: virthost16.phx2.fedoraproject.org +datacenter: phx2 From 4f2500ccd03006a7dee5a6e9f7d48d0f1139d29c Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:20:10 +0000 Subject: [PATCH 148/193] Conditionalize the denyhosts --- playbooks/groups/pkgs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index ff03341c79..47ec771809 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -28,7 +28,7 @@ roles: - base - rkhunter - - denyhosts + - { role: denyhosts, when: ansible_distribution_major_version != '7' } - nagios_client - fas_client - collectd/base From f86a5b188a0fa40a81542adb736dcaa694fa9058 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:26:29 +0000 Subject: [PATCH 149/193] Define some sizes for pkgs01.stg --- inventory/group_vars/pkgs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/pkgs b/inventory/group_vars/pkgs index c5789f3ba2..654dda260c 100644 --- a/inventory/group_vars/pkgs +++ b/inventory/group_vars/pkgs @@ -1,5 +1,7 @@ --- -# TODO: Define resources for this group of hosts here? +lvm_size: 100000 +mem_size: 4096 +num_cpus: 4 tcp_ports: [80, 443, 9418, # These 16 ports are used by fedmsg. One for each wsgi thread. From a6e6d2d16cf6f959fb3f8db2c3770ceb9d66ca73 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:43:57 +0000 Subject: [PATCH 150/193] Add missing hook file. --- roles/git/hooks/files/post-receive-fedmsg | 65 +++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 roles/git/hooks/files/post-receive-fedmsg diff --git a/roles/git/hooks/files/post-receive-fedmsg b/roles/git/hooks/files/post-receive-fedmsg new file mode 100644 index 0000000000..7bc9a140d9 --- /dev/null +++ b/roles/git/hooks/files/post-receive-fedmsg @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import getpass +import git +import os +import sys + +import fedmsg +import fedmsg.config + +# Read in all the rev information git-receive-pack hands us. +lines = [line.split() for line in sys.stdin.readlines()] + +# Use $GIT_DIR to determine where this repo is. +abspath = os.path.abspath(os.environ['GIT_DIR']) +repo_name = '.'.join(abspath.split(os.path.sep)[-1].split('.')[:-1]) + +username = getpass.getuser() + +repo = git.repo.Repo(abspath) +def _build_commit(rev): + old, rev, branch = rev + branch = '/'.join(branch.split('/')[2:]) + commit = repo.rev_parse(rev=rev) + + # We just don't handle these + if isinstance(commit, git.TagObject): + return None + + return dict( + name=commit.author.name, + email=commit.author.email, + username=username, + summary=commit.summary, + message=commit.message, + stats=dict( + files=commit.stats.files, + total=commit.stats.total, + ), + rev=rev, + path=abspath, + repo=repo_name, + branch=branch, + agent=os.getlogin(), + ) + +commits = map(_build_commit, lines) + +print "Emitting a message to the fedmsg bus." +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='scm', **config) + +for commit in commits: + + if commit is None: + continue + + fedmsg.publish( + # Expect this to change to just "receive" in the future. + topic="receive", + msg=dict(commit=commit), + modname="git", + ) From 05bef7f5077dfc5fd49d806ab39654f7c596a97a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:47:51 +0000 Subject: [PATCH 151/193] Add another hook --- roles/git/hooks/files/post-receive-chained | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 roles/git/hooks/files/post-receive-chained diff --git a/roles/git/hooks/files/post-receive-chained b/roles/git/hooks/files/post-receive-chained new file mode 100755 index 0000000000..b5c6e23112 --- /dev/null +++ b/roles/git/hooks/files/post-receive-chained @@ -0,0 +1,8 @@ +#!/bin/bash +# Redirect stdin to each of the post-receive hooks in place. + +# You need to explicitly add your hook to the following list +# for it to be invoked. +pee \ + $GIT_DIR/hooks/post-receive-chained.d/post-receive-email \ + $GIT_DIR/hooks/post-receive-chained.d/post-receive-fedmsg From 150c7e5c3cdb0ad0bae9bd3db707512dec9ade8b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 19:55:37 +0000 Subject: [PATCH 152/193] Add pkgs-stg group variables. --- inventory/group_vars/pkgs-stg | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 inventory/group_vars/pkgs-stg diff --git a/inventory/group_vars/pkgs-stg b/inventory/group_vars/pkgs-stg new file mode 100644 index 0000000000..654dda260c --- /dev/null +++ b/inventory/group_vars/pkgs-stg @@ -0,0 +1,47 @@ +--- +lvm_size: 100000 +mem_size: 4096 +num_cpus: 4 + +tcp_ports: [80, 443, 9418, + # These 16 ports are used by fedmsg. One for each wsgi thread. + 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, + 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] + +fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc +fas_client_restricted_app: /usr/bin/gl-auth-command +fas_client_admin_app: /usr/bin/gl-auth-command -s +fas_client_ssh_groups: "@cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc" + +git_group: packager +git_port: 9418 +git_server: /usr/libexec/git-core/git-daemon +git_server_args: --export-all --syslog --inetd --verbose +git_basepath: /srv/git/rpms + +clamscan_mailto: admin@fedoraproject.org +clamscan_paths: +- /srv/cache/lookaside/pkgs +clamscan_excludes: +- clamav- +- amavisd-new-2.3.3.tar.gz +- bro-20080804.tgz +- mailman- +- sagator- +- nicotine +- fwsnort-1.0.6.tar.gz +- psad-2.1.7.tar.bz2 +- pymilter- +- linkchecker- + +# These are consumed by a task in roles/fedmsg/base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: sysadmin +- service: scm + owner: root + group: packager +- service: lookaside + owner: root + group: apache From 27199cfa769316f4d5d43a6baa4a90df9b627f87 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:09:58 +0000 Subject: [PATCH 153/193] Move fedmsg stuff after apache install so we have an apache user. --- playbooks/groups/pkgs.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index 47ec771809..814487a22c 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -32,8 +32,6 @@ - nagios_client - fas_client - collectd/base - - fedmsg/base - - fedmsg/hub - sudo - git/hooks - git/make_checkout_seed @@ -54,3 +52,21 @@ handlers: - include: "{{ handlers }}/restart_services.yml" + +- name: setup fedmsg on pkgs + hosts: pkgs-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - fedmsg/base + - fedmsg/hub + + handlers: + - include: "{{ handlers }}/restart_services.yml" From b2ffffa3dfa83049393959308278aceb7cad5f23 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:14:38 +0000 Subject: [PATCH 154/193] Move some more stuff to the final play to be after yum repos are setup. --- playbooks/groups/pkgs.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index 814487a22c..d141775c4a 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -36,13 +36,6 @@ - git/hooks - git/make_checkout_seed - git/server - - gitolite/base - - gitolite/check_fedmsg_hooks - - cgit/base - - cgit/clean_lock_cron - - cgit/make_pkgs_list - - clamav - - distgit tasks: - include: "{{ tasks }}/yumrepos.yml" @@ -65,6 +58,13 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: + - gitolite/base + - gitolite/check_fedmsg_hooks + - cgit/base + - cgit/clean_lock_cron + - cgit/make_pkgs_list + - clamav + - distgit - fedmsg/base - fedmsg/hub From 27cfb06aa9d4368125666944463230f4e1f448ab Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:17:33 +0000 Subject: [PATCH 155/193] Revert "Move some more stuff to the final play to be after yum repos are setup." This reverts commit b2ffffa3dfa83049393959308278aceb7cad5f23. --- playbooks/groups/pkgs.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/playbooks/groups/pkgs.yml b/playbooks/groups/pkgs.yml index d141775c4a..814487a22c 100644 --- a/playbooks/groups/pkgs.yml +++ b/playbooks/groups/pkgs.yml @@ -36,6 +36,13 @@ - git/hooks - git/make_checkout_seed - git/server + - gitolite/base + - gitolite/check_fedmsg_hooks + - cgit/base + - cgit/clean_lock_cron + - cgit/make_pkgs_list + - clamav + - distgit tasks: - include: "{{ tasks }}/yumrepos.yml" @@ -58,13 +65,6 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: - - gitolite/base - - gitolite/check_fedmsg_hooks - - cgit/base - - cgit/clean_lock_cron - - cgit/make_pkgs_list - - clamav - - distgit - fedmsg/base - fedmsg/hub From 7f030966206f0e3404dd884ef1adeb79071c4671 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:20:26 +0000 Subject: [PATCH 156/193] Only gitolite3 on epel7 for now. --- roles/gitolite/base/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/gitolite/base/tasks/main.yml b/roles/gitolite/base/tasks/main.yml index b6c7916078..b241a1b26b 100644 --- a/roles/gitolite/base/tasks/main.yml +++ b/roles/gitolite/base/tasks/main.yml @@ -4,7 +4,7 @@ - name: install the needed packages yum: pkg={{item}} state=present with_items: - - gitolite + - gitolite3 - name: ensure the /etc/gitolite directory file: path=/etc/gitolite owner=root group=root mode=0755 state=directory From af2099be02e254ab7e82159d17b7a2825191c543 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Tue, 26 Aug 2014 20:23:17 +0000 Subject: [PATCH 157/193] add some meat to cloud-noc01 --- .../cloud-noc01.cloud.fedoraproject.org.yml | 3 + ...d.conf.cloud-noc01.cloud.fedoraproject.org | 339 ++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org diff --git a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml index 8d16831f03..2b2a318604 100644 --- a/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/cloud-noc01.cloud.fedoraproject.org.yml @@ -20,6 +20,8 @@ - fas_client - collectd/base - sudo + - dhcp_server + - tftp_server tasks: - include: "{{ tasks }}/yumrepos.yml" @@ -28,4 +30,5 @@ handlers: - include: "{{ handlers }}/restart_services.yml" + - include: "{{ handlers }}/semanage.yml" diff --git a/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org new file mode 100644 index 0000000000..5e88145dad --- /dev/null +++ b/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org @@ -0,0 +1,339 @@ +server-identifier cloud-noc01.cloud.fedoraproject.org; +ddns-update-style none; + +#subnet 172.16.1.0 netmask 255.255.255.0 { +#} +#subnet 172.16.2.0 netmask 255.255.255.0 { +#} +#subnet 172.16.3.0 netmask 255.255.255.0 { +#} +#subnet 172.16.4.0 netmask 255.255.255.0 { +#} +#subnet 172.16.5.0 netmask 255.255.255.0 { +#} +#subnet 172.16.6.0 netmask 255.255.255.0 { +#} +#subnet 172.16.7.0 netmask 255.255.255.0 { +#} +#subnet 172.16.8.0 netmask 255.255.255.0 { +#} +#subnet 172.16.9.0 netmask 255.255.255.0 { +#} + +# ARM MGMT VLAN +subnet 172.23.0.0 netmask 255.255.255.0 { + allow booting; + allow bootp; + + option domain-name "cloud.fedoraproject.org"; + option domain-name-servers 66.35.62.163, 152.19.134.150; + option routers 172.23.0.2; + option log-servers 172.23.0.2; + host arm03-packager00-mgmt { + hardware ethernet fc:2f:40:1b:64:4e; + fixed-address 172.23.0.70; + option host-name "arm03-packager00-mgmt"; + next-server 172.23.0.2; + } + host arm03-packager01-mgmt { + hardware ethernet fc:2f:40:08:d7:e2; + fixed-address 172.23.0.71; + option host-name "arm03-packager01-mgmt"; + next-server 172.23.0.2; + } + host arm03-qa00-mgmt { + hardware ethernet fc:2f:40:a1:f8:36; + fixed-address 172.23.0.72; + option host-name "arm03-qa00-mgmt"; + next-server 172.23.0.2; + } + host arm03-qa01-mgmt { + hardware ethernet fc:2f:40:1b:f6:da; + fixed-address 172.23.0.73; + option host-name "arm03-qa01-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc04-mgmt { + hardware ethernet fc:2f:40:16:c8:3a; + fixed-address 172.23.0.74; + option host-name "arm03-soc04-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc05-mgmt { + hardware ethernet fc:2f:40:9b:1d:da; + fixed-address 172.23.0.75; + option host-name "arm03-soc05-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc06-mgmt { + hardware ethernet fc:2f:40:00:4c:e2; + fixed-address 172.23.0.76; + option host-name "arm03-soc06-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc07-mgmt { + hardware ethernet fc:2f:40:57:cb:b6; + fixed-address 172.23.0.77; + option host-name "arm03-soc07-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc08-mgmt { + hardware ethernet fc:2f:40:12:6a:26; + fixed-address 172.23.0.78; + option host-name "arm03-soc08-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc09-mgmt { + hardware ethernet fc:2f:40:5d:85:8a; + fixed-address 172.23.0.79; + option host-name "arm03-soc09-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc10-mgmt { + hardware ethernet fc:2f:40:a5:5f:42; + fixed-address 172.23.0.80; + option host-name "arm03-soc10-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc11-mgmt { + hardware ethernet fc:2f:40:d2:a1:0e; + fixed-address 172.23.0.81; + option host-name "arm03-soc11-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc12-mgmt { + hardware ethernet fc:2f:40:00:a4:c6; + fixed-address 172.23.0.82; + option host-name "arm03-soc12-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc13-mgmt { + hardware ethernet fc:2f:40:59:27:ba; + fixed-address 172.23.0.83; + option host-name "arm03-soc13-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc14-mgmt { + hardware ethernet fc:2f:40:7b:ab:f6; + fixed-address 172.23.0.84; + option host-name "arm03-soc14-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc15-mgmt { + hardware ethernet fc:2f:40:8a:99:96; + fixed-address 172.23.0.85; + option host-name "arm03-soc15-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc16-mgmt { + hardware ethernet fc:2f:40:ea:ff:16; + fixed-address 172.23.0.86; + option host-name "arm03-soc16-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc17-mgmt { + hardware ethernet fc:2f:40:79:e3:8e; + fixed-address 172.23.0.87; + option host-name "arm03-soc17-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc18-mgmt { + hardware ethernet fc:2f:40:7d:0c:9a; + fixed-address 172.23.0.88; + option host-name "arm03-soc18-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc19-mgmt { + hardware ethernet fc:2f:40:04:29:9a; + fixed-address 172.23.0.89; + option host-name "arm03-soc19-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc20-mgmt { + hardware ethernet fc:2f:40:3c:50:26; + fixed-address 172.23.0.90; + option host-name "arm03-soc20-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc21-mgmt { + hardware ethernet fc:2f:40:cb:4f:66; + fixed-address 172.23.0.91; + option host-name "arm03-soc21-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc22-mgmt { + hardware ethernet fc:2f:40:17:c0:ee; + fixed-address 172.23.0.92; + option host-name "arm03-soc22-mgmt"; + next-server 172.23.0.2; + } + host arm03-soc23-mgmt { + hardware ethernet fc:2f:40:00:47:3e; + fixed-address 172.23.0.93; + option host-name "arm03-soc23-mgmt"; + next-server 172.23.0.2; + } +} + +# ARM VLAN +subnet 209.132.184.0 netmask 255.255.255.0 { + allow booting; + allow bootp; + + option domain-name "cloud.fedoraproject.org fedoraproject.org"; + option domain-name-servers 66.35.62.163, 152.19.134.150; + option routers 209.132.184.254; + option log-servers 209.132.184.2; + host fed-cloud09 { + hardware ethernet f0:1f:af:e3:5f:0c; + fixed-address 209.132.184.9; + option host-name "fed-cloud09.cloud.fedoraproject.org"; + next-server 209.132.184.2; + filename "pxelinux.0"; + } + host arm03-packager00 { + hardware ethernet fc:2f:40:1b:64:4c; + fixed-address 209.132.184.70; + option host-name "arm03-packager00"; + next-server 209.132.184.2; + } + host arm03-packager01 { + hardware ethernet fc:2f:40:08:d7:e0; + fixed-address 209.132.184.71; + option host-name "arm03-packager01"; + next-server 209.132.184.2; + } + host arm03-qa00 { + hardware ethernet fc:2f:40:a1:f8:34; + fixed-address 209.132.184.72; + option host-name "arm03-qa00"; + next-server 209.132.184.2; + } + host arm03-qa01 { + hardware ethernet fc:2f:40:1b:f6:d8; + fixed-address 209.132.184.73; + option host-name "arm03-qa01"; + next-server 209.132.184.2; + } + host arm03-soc04 { + hardware ethernet fc:2f:40:16:c8:38; + fixed-address 209.132.184.74; + option host-name "arm03-soc04"; + next-server 209.132.184.2; + } + host arm03-soc05 { + hardware ethernet fc:2f:40:9b:1d:d8; + fixed-address 209.132.184.75; + option host-name "arm03-soc05"; + next-server 209.132.184.2; + } + host arm03-soc06 { + hardware ethernet fc:2f:40:00:4c:e0; + fixed-address 209.132.184.76; + option host-name "arm03-soc06"; + next-server 209.132.184.2; + } + host arm03-soc07 { + hardware ethernet fc:2f:40:57:cb:b4; + fixed-address 209.132.184.77; + option host-name "arm03-soc07"; + next-server 209.132.184.2; + } + host arm03-soc08 { + hardware ethernet fc:2f:40:12:6a:24; + fixed-address 209.132.184.78; + option host-name "arm03-soc08"; + next-server 209.132.184.2; + } + host arm03-soc09 { + hardware ethernet fc:2f:40:5d:85:88; + fixed-address 209.132.184.79; + option host-name "arm03-soc09"; + next-server 209.132.184.2; + } + host arm03-soc10 { + hardware ethernet fc:2f:40:a5:5f:40; + fixed-address 209.132.184.80; + option host-name "arm03-soc10"; + next-server 209.132.184.2; + } + host arm03-soc11 { + hardware ethernet fc:2f:40:d2:a1:0c; + fixed-address 209.132.184.81; + option host-name "arm03-soc11"; + next-server 209.132.184.2; + } + host arm03-soc12 { + hardware ethernet fc:2f:40:00:a4:c4; + fixed-address 209.132.184.82; + option host-name "arm03-soc12"; + next-server 209.132.184.2; + } + host arm03-soc13 { + hardware ethernet fc:2f:40:59:27:b8; + fixed-address 209.132.184.83; + option host-name "arm03-soc13"; + next-server 209.132.184.2; + } + host arm03-soc14 { + hardware ethernet fc:2f:40:7b:ab:f4; + fixed-address 209.132.184.84; + option host-name "arm03-soc14"; + next-server 209.132.184.2; + } + host arm03-soc15 { + hardware ethernet fc:2f:40:8a:99:94; + fixed-address 209.132.184.85; + option host-name "arm03-soc15"; + next-server 209.132.184.2; + } + host arm03-soc16 { + hardware ethernet fc:2f:40:ea:ff:14; + fixed-address 209.132.184.86; + option host-name "arm03-soc16"; + next-server 209.132.184.2; + } + host arm03-soc17 { + hardware ethernet fc:2f:40:79:e3:8c; + fixed-address 209.132.184.87; + option host-name "arm03-soc17"; + next-server 209.132.184.2; + } + host arm03-soc18 { + hardware ethernet fc:2f:40:7d:0c:98; + fixed-address 209.132.184.88; + option host-name "arm03-soc18"; + next-server 209.132.184.2; + } + host arm03-soc19 { + hardware ethernet fc:2f:40:04:29:98; + fixed-address 209.132.184.89; + option host-name "arm03-soc19"; + next-server 209.132.184.2; + } + host arm03-soc20 { + hardware ethernet fc:2f:40:3c:50:24; + fixed-address 209.132.184.90; + option host-name "arm03-soc20"; + next-server 209.132.184.2; + } + host arm03-soc21 { + hardware ethernet fc:2f:40:cb:4f:64; + fixed-address 209.132.184.91; + option host-name "arm03-soc21"; + next-server 209.132.184.2; + } + host arm03-soc22 { + hardware ethernet fc:2f:40:17:c0:ec; + fixed-address 209.132.184.92; + option host-name "arm03-soc22"; + next-server 209.132.184.2; + } + host arm03-soc23 { + hardware ethernet fc:2f:40:00:47:3c; + fixed-address 209.132.184.93; + option host-name "arm03-soc23"; + next-server 209.132.184.2; + } + +} From 7f75064348bf41241d1eb9e4ddf39ccdf42dd628 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:25:24 +0000 Subject: [PATCH 158/193] Add check-perms script --- .../check_fedmsg_hooks/files/check-perms.py | 419 ++++++++++++++++++ 1 file changed, 419 insertions(+) create mode 100755 roles/gitolite/check_fedmsg_hooks/files/check-perms.py diff --git a/roles/gitolite/check_fedmsg_hooks/files/check-perms.py b/roles/gitolite/check_fedmsg_hooks/files/check-perms.py new file mode 100755 index 0000000000..31d4215672 --- /dev/null +++ b/roles/gitolite/check_fedmsg_hooks/files/check-perms.py @@ -0,0 +1,419 @@ +#!/usr/bin/python -tt +"""Check permissions of a tree of git repositories, optionally fixing any +problems found. +""" + +import os +import re +import sys +import optparse +from stat import * +from subprocess import call, PIPE, Popen + +ALL_CHECKS = ['bare', 'shared', 'mail-hook', 'fedmsg-hook', 'perms', 'post-update-hook'] +DEFAULT_CHECKS = ['bare', 'shared', 'perms', 'post-update-hook'] + +OBJECT_RE = re.compile('[0-9a-z]{40}') + +def error(msg): + print >> sys.stderr, msg + +def is_object(path): + """Check if a path is a git object.""" + parts = path.split(os.path.sep) + if 'objects' in parts and len(parts) > 2 and \ + OBJECT_RE.match(''.join(path.split(os.path.sep)[-2:])): + return True + return False + +def is_bare_repo(gitdir): + """Check if a git repository is bare.""" + cmd = ['git', '--git-dir', gitdir, 'config', '--bool', 'core.bare'] + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + bare, error = p.communicate() + if bare.rstrip() != 'true' or p.returncode: + return False + return True + +def is_shared_repo(gitdir): + """Check if a git repository is shared.""" + cmd = ['git', '--git-dir', gitdir, 'config', 'core.sharedRepository'] + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + shared, error = p.communicate() + sharedmodes = ['1', 'group', 'true', '2', 'all', 'world', 'everybody'] + if shared.rstrip() not in sharedmodes or p.returncode: + return False + return True + +def uses_version1_mail_hook(gitdir): + """Check if a git repository uses the old fedora-git-commit-mail-hook.""" + hook = os.path.join(gitdir, 'hooks/update') + oldpath = '/usr/bin/fedora-git-commit-mail-hook' + return os.path.realpath(hook) == oldpath + +def uses_version2_mail_hook(gitdir): + """Check if a git repository uses the pre-fedmsg mail-hook setup.""" + hook = os.path.join(gitdir, 'hooks/post-receive') + oldpath = '/usr/share/git-core/mail-hooks/gnome-post-receive-email' + return os.path.realpath(hook) == oldpath + +def check_post_update_hook(gitdir, fix=False): + """Check if a repo's post-update hook is setup correctly.""" + hook = os.path.join(gitdir, 'hooks/post-update') + realpath = os.path.realpath(hook) + goodpath = '/usr/share/git-core/templates/hooks/post-update.sample' + badpath = '/usr/bin/git-update-server-info' + + if realpath == goodpath: + return True + + errmsg = '' + if realpath == badpath: + errmsg = '%s: symlinked to %s' % (hook, badpath) + elif not os.path.exists(hook): + errmsg = '%s: does not exist' % hook + elif not os.access(hook, os.X_OK): + errmsg = '%s: is not executable' % hook + elif not os.path.islink(hook): + errmsg = '%s: not a symlink' % hook + else: + errmsg = '%s: symlinked to %s' % (hook, realpath) + + error(errmsg) + + if not fix: + return False + + if not os.path.exists(goodpath): + error('%s: post-update hook (%s) does not exist.' % (gitdir, goodpath)) + return False + + if os.path.exists(hook): + try: + os.rename(hook, '%s~' % hook) + except (IOError, OSError), err: + error('%s: Error renaming %s: %s' % (gitdir, hook, err.strerror)) + return False + try: + os.symlink(goodpath, hook) + except (IOError, OSError), err: + error('%s: Error creating %s symlink: %s' % (gitdir, hook, err.strerror)) + return False + + return True + +def set_bare_repo(gitdir): + """Set core.bare for a git repository.""" + cmd = ['git', '--git-dir', gitdir, 'config', '--bool', 'core.bare', 'true'] + ret = call(cmd) + if ret: + return False + return True + +def set_shared_repo(gitdir, value='group'): + """Set core.sharedRepository for a git repository.""" + mode_re = re.compile('06[0-7]{2}') + if value in [0, 'false', 'umask']: + value = 'umask' + elif value in [1, 'true', 'group']: + value = 'group' + elif value in [2, 'all', 'world', 'everybody']: + value = 'all' + elif mode_re.match(value): + pass + else: + raise SystemExit('Bogus core.sharedRepository value "%s"' % value) + cmd = ['git', '--git-dir', gitdir, 'config', 'core.sharedRepository', + value] + ret = call(cmd) + if ret: + return False + return True + + +def set_post_receive_hook_version2(gitdir): + """Configure a git repository to use the gnome mail hook without fedmsg.""" + + # Get recipients from the commit-list file. + commit_list = os.path.join(gitdir, 'commit-list') + if not os.path.exists(commit_list): + error('%s: No commit-list file found' % gitdir) + return False + try: + addrs = open(commit_list).read().strip() + addrs = ', '.join(addrs.split()) + except: + error('%s: Unable to read commit-list file' % gitdir) + return False + + # Set hooks.mailinglist + if '@' not in addrs: + addrs = '%s@lists.fedorahosted.org' + cmd = ['git', '--git-dir', gitdir, 'config', 'hooks.mailinglist', addrs] + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.returncode: + error('%s: Error setting hooks.mailinglist: %s' % (gitdir, stderr)) + return False + + # Set hooks.maildomain + cmd = ['git', '--git-dir', gitdir, 'config', 'hooks.maildomain', + 'fedoraproject.org'] + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + if p.returncode: + error('%s: Error setting hooks.maildomain: %s' % (gitdir, stderr)) + return False + + # Symlink mail notification script to post-receive hook + script = '/usr/share/git-core/mail-hooks/gnome-post-receive-email' + if not os.path.exists(script): + error('%s: Mail hook (%s) does not exist.' % (gitdir, script)) + return False + + hook = os.path.join(gitdir, 'hooks', 'post-receive') + if os.path.exists(hook): + try: + os.remove(hook) + except Exception, e: + errstr = hasattr(e, 'strerror') and e.strerror or e + error('%s: Error removing %s: %s' % (gitdir, hook, errstr)) + return False + try: + os.symlink(script, hook) + except Exception, e: + errstr = hasattr(e, 'strerror') and e.strerror or e + error('%s: Error creating %s symlink: %s' % (gitdir, hook, errstr)) + return False + + # Clean up commit-list file and old update hook link + try: + os.rename(commit_list, '%s~' % commit_list) + except (IOError, OSError), err: + error('%s: Unable to backup commit-list: %s' % (gitdir, err.strerror)) + return False + try: + oldhook = os.path.join(gitdir, 'hooks/update') + os.remove(oldhook) + except (IOError, OSError), err: + error('%s: Unable to backup commit-list: %s' % (gitdir, err.strerror)) + return False + + # We ran the gauntlet. + return True + + +def set_post_receive_hook_version3(gitdir): + """Configure a git repository to use the fedmsg+gnome-mail hooks.""" + + if not uses_version2_mail_hook(gitdir): + error('%s: Not yet on version2 mail hook; do --fix=mail-hook' % gitdir) + return False + + # Check that the destination is 'okay' + dest_prefix = os.path.join(gitdir, 'hooks', 'post-receive-chained.d') + + if not os.path.exists(dest_prefix): + os.mkdir(dest_prefix) + + if not os.path.isdir(dest_prefix): + error('%s: %s is not a directory.' % (gitdir, dest_prefix)) + return False + + # Symlink mail notification and fedmsg scripts to post-receive hook + scripts = { + '/usr/share/git-core/mail-hooks/gnome-post-receive-email': + os.path.join(dest_prefix, 'post-receive-email'), + '/usr/share/git-core/post-receive-fedmsg': + os.path.join(dest_prefix, 'post-receive-fedmsg'), + + # This one kicks off all the others. + '/usr/share/git-core/post-receive-chained': + os.path.join(gitdir, 'hooks', 'post-receive'), + + } + + for script, hook in scripts.items(): + if not os.path.exists(script): + error('%s: Hook (%s) does not exist.' % (gitdir, script)) + return False + + if os.path.exists(hook): + try: + os.remove(hook) + except Exception, e: + errstr = hasattr(e, 'strerror') and e.strerror or e + error('%s: Error removing %s: %s' % (gitdir, hook, errstr)) + return False + try: + os.symlink(script, hook) + except Exception, e: + errstr = hasattr(e, 'strerror') and e.strerror or e + error('%s: Error creating %s symlink: %s' % (gitdir, hook, errstr)) + return False + + # We ran the gauntlet. + return True + + +def list_checks(): + print 'Available checks: %s' % ', '.join(ALL_CHECKS) + print 'Default checks: %s' % ', '.join(DEFAULT_CHECKS) + + +def check_git_perms(path, fix=False): + """Check if permissions on a git repo are correct. + + If fix is true, problems found are corrected. + """ + object_mode = S_IRUSR | S_IRGRP | S_IROTH + oldmode = mode = S_IMODE(os.lstat(path)[ST_MODE]) + errors = [] + if os.path.isdir(path): + newmode = mode | S_ISGID + if mode != newmode: + msg = 'Not SETGID (should be "%s")' % oct(newmode) + errors.append(msg) + mode = newmode + elif is_object(path) and mode ^ object_mode: + msg = 'Wrong object mode "%s" (should be "%s")' % ( + oct(mode), oct(object_mode)) + errors.append(msg) + mode = object_mode + if mode & S_IWUSR and not is_object(path): + newmode = mode | S_IWGRP + exempt = \ + any(map(path.endswith, ['commit-list', 'gl-conf'])) or \ + any(map(path.__contains__, ['/hooks/'])) + + if mode != newmode and not exempt: + msg = 'Not group writable (should be "%s")' % oct(newmode) + errors.append(msg) + mode = newmode + if mode != oldmode and not os.path.islink(path): + errmsg = '%s:' % path + errmsg += ', '.join(['%s' % e for e in errors]) + error(errmsg) + if not fix: + return False + try: + os.chmod(path, mode) + return True + except Exception, e: + errstr = hasattr(e, 'strerror') and e.strerror or e + mode = oct(mode) + error('%s: Error setting "%s" mode on %s: %s' % (gitdir, + mode, path, errstr)) + return False + return True + +def main(): + usage = '%prog [options] [gitroot]' + parser = optparse.OptionParser(usage=usage) + parser.add_option('-f', '--fix', action='store_true', default=False, + help='Correct any problems [%default]') + parser.add_option('-l', '--list-checks', action='store_true', + help='List default checks') + parser.add_option('-c', '--check', dest='checks', action='append', + default=[], metavar='check', + help='Add a check, may be used multiple times') + parser.add_option('-s', '--skip', action='append', default=[], + metavar='check', + help='Skip a check, may be used multiple times') + parser.add_option('-r', '--repo', default=None, + help="Check only a certain repo, not all of them.") + opts, args = parser.parse_args() + + # Check options + if opts.list_checks: + list_checks() + raise SystemExit + + if opts.checks: + checks = set(opts.checks) + bad_check_opts = checks.difference(set(ALL_CHECKS)) + if bad_check_opts: + msg = 'Bad check(s): %s' % ', '.join(sorted(bad_check_opts)) + msg += '\nAvailable checks: %s' % ', '.join(ALL_CHECKS) + raise SystemExit(msg) + else: + bad_skip_opts = set(opts.skip).difference(set(ALL_CHECKS)) + if bad_skip_opts: + msg = 'Bad skip option(s): %s' % ', '.join(sorted(bad_skip_opts)) + msg += '\nAvailable checks: %s' % ', '.join(ALL_CHECKS) + raise SystemExit(msg) + checks = set() + for check in DEFAULT_CHECKS: + if check not in opts.skip: + checks.add(check) + + # Check args + if len(args) > 1: + raise SystemExit(parser.get_usage().strip()) + + gitroot = args and args[0] or '/git' + + if not os.path.isdir(gitroot): + raise SystemExit('%s does not exist or is not a directory' % gitroot) + + if opts.repo: + gitdirs = ['/'.join([gitroot, opts.repo])] + else: + gitdirs = [] + for path, dirs, files in os.walk(gitroot): + if path in gitdirs: + continue + if 'description' in os.listdir(path): + gitdirs.append(path) + + problems = [] + for gitdir in sorted(gitdirs): + if 'bare' in checks and not is_bare_repo(gitdir): + error('%s: core.bare not true' % gitdir) + if not opts.fix or not set_bare_repo(gitdir): + problems.append(gitdir) + if 'shared' in checks and not is_shared_repo(gitdir): + error('%s: core.sharedRepository not set' % gitdir) + if not opts.fix or not set_shared_repo(gitdir): + problems.append(gitdir) + + if 'mail-hook' in checks and uses_version1_mail_hook(gitdir): + error('%s: uses old mail hook' % gitdir) + if not opts.fix or not set_post_receive_hook_version2(gitdir): + problems.append(gitdir) + + if 'fedmsg-hook' in checks and (uses_version1_mail_hook(gitdir) or + uses_version2_mail_hook(gitdir)): + error('%s: uses the gnome mail hook or older' % gitdir) + if not opts.fix or not set_post_receive_hook_version3(gitdir): + problems.append(gitdir) + + if 'post-update-hook' in checks and not check_post_update_hook(gitdir, + opts.fix): + problems.append(gitdir) + + if 'perms' in checks: + paths = [] + for path, dirs, files in os.walk(gitdir): + for d in dirs: + d = os.path.join(path, d) + if d not in paths: + paths.append(d) + for f in files: + f = os.path.join(path, f) + if f not in paths: + paths.append(f) + for path in paths: + if not check_git_perms(path, fix=opts.fix): + problems.append(path) + + if problems: + raise SystemExit('%d problems remain unfixed' % len(problems)) + + raise SystemExit() + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + raise SystemExit('\nExiting on user cancel (Ctrl-C)') From f5c4d05f60cfc9da7a36f163f4dba858dacdeb0a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:32:17 +0000 Subject: [PATCH 159/193] No lua-md5 on epel7 yet. ;( --- roles/cgit/base/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/cgit/base/tasks/main.yml b/roles/cgit/base/tasks/main.yml index 3e7b810fe4..1ac9239a83 100644 --- a/roles/cgit/base/tasks/main.yml +++ b/roles/cgit/base/tasks/main.yml @@ -5,7 +5,8 @@ yum: pkg={{item}} state=present with_items: - cgit - - lua-md5 +# https://bugzilla.redhat.com/show_bug.cgi?id=1134103 +# - lua-md5 # See https://github.com/robyduck/cgit-custom - name: Make sure cgit directory exists From 7f2b5e0e529571450e8edcab40097124dd5210ee Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:41:01 +0000 Subject: [PATCH 160/193] Move image files to the right place. --- roles/cgit/base/files/{ => images}/favicon.ico | Bin roles/cgit/base/files/{ => images}/html-bg.png | Bin roles/cgit/base/files/{ => images}/t.png | Bin 3 files changed, 0 insertions(+), 0 deletions(-) rename roles/cgit/base/files/{ => images}/favicon.ico (100%) rename roles/cgit/base/files/{ => images}/html-bg.png (100%) rename roles/cgit/base/files/{ => images}/t.png (100%) diff --git a/roles/cgit/base/files/favicon.ico b/roles/cgit/base/files/images/favicon.ico similarity index 100% rename from roles/cgit/base/files/favicon.ico rename to roles/cgit/base/files/images/favicon.ico diff --git a/roles/cgit/base/files/html-bg.png b/roles/cgit/base/files/images/html-bg.png similarity index 100% rename from roles/cgit/base/files/html-bg.png rename to roles/cgit/base/files/images/html-bg.png diff --git a/roles/cgit/base/files/t.png b/roles/cgit/base/files/images/t.png similarity index 100% rename from roles/cgit/base/files/t.png rename to roles/cgit/base/files/images/t.png From 43da2aabafbd4c7dec6b0c3f43df878fe2cada28 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:45:10 +0000 Subject: [PATCH 161/193] Tweak clamav package names --- roles/clamav/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml index 0bf8bab8f8..3f5730e1fe 100644 --- a/roles/clamav/tasks/main.yml +++ b/roles/clamav/tasks/main.yml @@ -17,7 +17,8 @@ yum: pkg={{item}} state=present with_items: - clamav - - clamav-db + - clamav-data + - clamav-update - cronie-anacron - name: setup the freshclam configuration From 63b2de4aa20cf271f4b01a2e30392dbd09c9e470 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:50:30 +0000 Subject: [PATCH 162/193] Try this fix for template --- roles/clamav/templates/clamscan.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/clamav/templates/clamscan.sh.j2 b/roles/clamav/templates/clamscan.sh.j2 index 324b646e72..8b5d957f9f 100644 --- a/roles/clamav/templates/clamscan.sh.j2 +++ b/roles/clamav/templates/clamscan.sh.j2 @@ -17,7 +17,7 @@ find {{ path }} -ctime -${DAYS} -type f >> $FILELIST /bin/nice -5 /usr/bin/clamscan --infected --quiet --file-list=$FILELIST --log=$LOGFILE \ {% for path in clamscan_excludes|default([]) %} ---exclude={{ exclude }} \ +--exclude={{ path }} \ {% endfor %} 2> /dev/null From cf00215db0f22f2e77ca70dbb742ef377ddb3046 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:54:41 +0000 Subject: [PATCH 163/193] Add some quotes. --- roles/clamav/tasks/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml index 3f5730e1fe..2b26dc50d6 100644 --- a/roles/clamav/tasks/main.yml +++ b/roles/clamav/tasks/main.yml @@ -36,10 +36,10 @@ - name: setup cron job for clam scan cron: > name="clamscan" - minute={{ clamscan_minute | default(35) }} - hour={{ clamscan_hour | default(4) }} - day={{ clamscan_day | default(*) }} - month={{ clamscan_month | default(*) }} - weekday={{ clamscan_weekday | default(5) }} + minute="{{ clamscan_minute | default(35) }}" + hour="{{ clamscan_hour | default(4) }}" + day="{{ clamscan_day | default(*) }}" + month="{{ clamscan_month | default(*) }}" + weekday="{{ clamscan_weekday | default(5) }}" job="/usr/local/bin/clamscan.sh" cron_file="ansible-clamscan" From 74f56aa6f3c591eb2b60c6516ea3c16f3fc437d0 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 20:59:37 +0000 Subject: [PATCH 164/193] Need a user= here. --- roles/clamav/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml index 2b26dc50d6..bd5c288885 100644 --- a/roles/clamav/tasks/main.yml +++ b/roles/clamav/tasks/main.yml @@ -36,6 +36,7 @@ - name: setup cron job for clam scan cron: > name="clamscan" + user=root minute="{{ clamscan_minute | default(35) }}" hour="{{ clamscan_hour | default(4) }}" day="{{ clamscan_day | default(*) }}" From d87f8a52ee03123a0d8b8e3ac1fc33bebc06b1c2 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 21:05:50 +0000 Subject: [PATCH 165/193] Don't need to add group here, it's in fas --- roles/distgit/tasks/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 240c5c0e03..7249928f5c 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -13,13 +13,6 @@ - mod_ssl - python-fedmsg-genacls -- name: set some sysctl options - sysctl: name=vm.zone_reclaim_mode value=1 state=present - when: env != "staging" - -- name: create the packager group - group: name=packager gid=415 state=present - - name: install the httpd config file copy: src=pkgs.fedoraproject.org.conf dest=/etc/httpd/conf.d/pkgs.fedoraproject.org.conf notify: From 89a860d3b666bd7897672ca08c4e449716eefa85 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 21:13:44 +0000 Subject: [PATCH 166/193] Correct path --- roles/distgit/tasks/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 7249928f5c..5018c40350 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -127,9 +127,11 @@ - name: create the Lookaside Cache root directory file: dest=/srv/cache/lookaside/pkgs state=directory -- name: install the certificates - copy: src={{private}}/fedora-ca.cert dest=/etc/httpd/conf/cacert.pem - copy: src={{private}}/pkgs.fedoraproject.org_key_and_cert.pem dest=/etc/httpd/conf/pkgs.fedoraproject.org_key_and_cert.pem owner=apache mode=0400 +- name: install the fedora-ca.cert + copy: src={{private}}/files/fedora-ca.cert dest=/etc/httpd/conf/cacert.pem + +- name: install the pkgs cert + copy: src={{private}}/files/pkgs.fedoraproject.org_key_and_cert.pem dest=/etc/httpd/conf/pkgs.fedoraproject.org_key_and_cert.pem owner=apache mode=0400 - name: install the updatecrl.sh script copy: src=updatecrl.sh dest=/usr/local/bin/updatecrl.sh owner=root mode=755 From 278cd376ea9d5fb93fb1b91dd959dacf131a6eb8 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 21:19:00 +0000 Subject: [PATCH 167/193] Add dist-git-upload cgi --- roles/distgit/files/dist-git-upload.cgi | 219 ++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 roles/distgit/files/dist-git-upload.cgi diff --git a/roles/distgit/files/dist-git-upload.cgi b/roles/distgit/files/dist-git-upload.cgi new file mode 100644 index 0000000000..3e8cc97dcb --- /dev/null +++ b/roles/distgit/files/dist-git-upload.cgi @@ -0,0 +1,219 @@ +#!/usr/bin/python +# +# CGI script to handle file updates for the rpms git repository. There +# is nothing really complex here other than tedious checking of our +# every step along the way... +# +# License: GPL + +import os +import sys +import cgi +import tempfile +import grp +import pwd +import syslog +import smtplib + +import fedmsg +import fedmsg.config + +from email import Header, Utils +try: + from email.mime.text import MIMEText +except ImportError: + from email.MIMEText import MIMEText + +try: + import hashlib + md5_constructor = hashlib.md5 +except ImportError: + import md5 + md5_constructor = md5.new + +# Reading buffer size +BUFFER_SIZE = 4096 + +# We check modules exist from this dircetory +GITREPO = '/srv/git/rpms' + +# Lookaside cache directory +CACHE_DIR = '/srv/cache/lookaside/pkgs' + +# Fedora Packager Group +PACKAGER_GROUP = 'packager' + +def send_error(text): + print text + sys.exit(1) + +def check_form(form, var): + ret = form.getvalue(var, None) + if ret is None: + send_error('Required field "%s" is not present.' % var) + if isinstance(ret, list): + send_error('Multiple values given for "%s". Aborting.' % var) + return ret + +def check_auth(username): + authenticated = False + try: + if username in grp.getgrnam(PACKAGER_GROUP)[3]: + authenticated = True + except KeyError: + pass + return authenticated + +def send_email(pkg, md5, filename, username): + text = """A file has been added to the lookaside cache for %(pkg)s: + +%(md5)s %(filename)s""" % locals() + msg = MIMEText(text) + try: + sender_name = pwd.getpwnam(username)[4] + sender_email = '%s@fedoraproject.org' % username + except KeyError: + sender_name = '' + sender_email = 'nobody@fedoraproject.org' + syslog.syslog('Unable to find account info for %s (uploading %s)' % + (username, filename)) + if sender_name: + try: + sender_name = unicode(sender_name, 'ascii') + except UnicodeDecodeError: + sender_name = Header.Header(sender_name, 'utf-8').encode() + msg.set_charset('utf-8') + sender = Utils.formataddr((sender_name, sender_email)) + recipients = ['%s-owner@fedoraproject.org' % pkg, + 'scm-commits@lists.fedoraproject.org'] + msg['Subject'] = 'File %s uploaded to lookaside cache by %s' % ( + filename, username) + msg['From'] = sender + msg['To'] = ', '.join(recipients) + msg['X-Fedora-Upload'] = '%s, %s' % (pkg, filename) + try: + s = smtplib.SMTP('bastion') + s.sendmail(sender, recipients, msg.as_string()) + except: + syslog.syslog('sending mail for upload of %s failed!' % filename) + +def main(): + os.umask(002) + + username = os.environ.get('SSL_CLIENT_S_DN_CN', None) + if not check_auth(username): + print 'Status: 403 Forbidden' + print 'Content-type: text/plain' + print + print 'You must connect with a valid certificate and be in the %s group to upload.' % PACKAGER_GROUP + sys.exit(0) + + print 'Content-Type: text/plain' + print + + assert os.environ['REQUEST_URI'].split('/')[1] == 'repo' + + form = cgi.FieldStorage() + name = check_form(form, 'name') + md5sum = check_form(form, 'md5sum') + + action = None + upload_file = None + filename = None + + # Is this a submission or a test? + # in a test, we don't get a file, just a filename. + # In a submission, we don;t get a filename, just the file. + if form.has_key('filename'): + action = 'check' + filename = check_form(form, 'filename') + filename = os.path.basename(filename) + print >> sys.stderr, '[username=%s] Checking file status: NAME=%s FILENAME=%s MD5SUM=%s' % (username, name, filename, md5sum) + else: + action = 'upload' + if form.has_key('file'): + upload_file = form['file'] + if not upload_file.file: + send_error('No file given for upload. Aborting.') + filename = os.path.basename(upload_file.filename) + else: + send_error('Required field "file" is not present.') + print >> sys.stderr, '[username=%s] Processing upload request: NAME=%s FILENAME=%s MD5SUM=%s' % (username, name, filename, md5sum) + + module_dir = os.path.join(CACHE_DIR, name) + md5_dir = os.path.join(module_dir, filename, md5sum) + + # first test if the module really exists + git_dir = os.path.join(GITREPO, '%s.git' % name) + if not os.path.isdir(git_dir): + print >> sys.stderr, '[username=%s] Unknown module: %s' % (username, name) + send_error('Module "%s" does not exist!' % name) + + # try to see if we already have this file... + dest_file = os.path.join(md5_dir, filename) + if os.path.exists(dest_file): + if action == 'check': + print 'Available' + else: + upload_file.file.close() + dest_file_stat = os.stat(dest_file) + print 'File %s already exists' % filename + print 'File: %s Size: %d' % (dest_file, dest_file_stat.st_size) + sys.exit(0) + elif action == 'check': + print 'Missing' + sys.exit(0) + + # check that all directories are in place + if not os.path.isdir(module_dir): + os.makedirs(module_dir, 02775) + + # grab a temporary filename and dump our file in there + tempfile.tempdir = module_dir + tmpfile = tempfile.mkstemp(md5sum)[1] + tmpfd = open(tmpfile, 'w') + + # now read the whole file in + m = md5_constructor() + filesize = 0 + while True: + data = upload_file.file.read(BUFFER_SIZE) + if not data: + break + tmpfd.write(data) + m.update(data) + filesize += len(data) + + # now we're done reading, check the MD5 sum of what we got + tmpfd.close() + check_md5sum = m.hexdigest() + if md5sum != check_md5sum: + send_error("MD5 check failed. Received %s instead of %s." % (check_md5sum, md5sum)) + + # wow, even the MD5SUM matches. make sure full path is valid now + if not os.path.isdir(md5_dir): + os.makedirs(md5_dir, 02775) + print >> sys.stderr, '[username=%s] mkdir %s' % (username, md5_dir) + + os.rename(tmpfile, dest_file) + os.chmod(dest_file, 0644) + + print >> sys.stderr, '[username=%s] Stored %s (%d bytes)' % (username, dest_file, filesize) + print 'File %s size %d MD5 %s stored OK' % (filename, filesize, md5sum) + send_email(name, md5sum, filename, username) + + # Emit a fedmsg message. Load the config to talk to the fedmsg-relay. + try: + config = fedmsg.config.load_config([], None) + config['active'] = True + config['endpoints']['relay_inbound'] = config['relay_inbound'] + fedmsg.init(name="relay_inbound", cert_prefix="lookaside", **config) + + topic = "lookaside.new" + msg = dict(name=name, md5sum=md5sum, filename=filename, agent=username) + fedmsg.publish(modname="git", topic=topic, msg=msg) + except Exception as e: + print "Error with fedmsg", str(e) + +if __name__ == '__main__': + main() From efe0ebf7d958fb55540f648e1d7e01a028517919 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 21:23:50 +0000 Subject: [PATCH 168/193] Add a /srv/web dir --- roles/distgit/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 5018c40350..94e487ef3c 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -146,6 +146,9 @@ user=root job="/usr/local/bin/updatecrl.sh" +- name: create /srv/web directory + file: dest=/srv/web state=directory + - name: install the upload CGI script copy: src=dist-git-upload.cgi dest=/srv/web/upload.cgi owner=root group=root mode=0755 notify: From 241b166ac40ffdb39372736b39cb9dde6c28c5c2 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 21:26:13 +0000 Subject: [PATCH 169/193] Fix path --- tasks/drbackupkey.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/drbackupkey.yml b/tasks/drbackupkey.yml index 0f2741fd80..2f71fbcdf4 100644 --- a/tasks/drbackupkey.yml +++ b/tasks/drbackupkey.yml @@ -12,7 +12,7 @@ file: dest=/var/lib/drbackup/.ssh/ state=directory owner=drbackup group=drbackup mode=0700 - name: install the backup ssh keys - copy: src={{private}}/backup.pub dest=/var/lib/drbackup/.ssh/authorized_keys owner=drbackup group=drbackup mode=0600 + copy: src={{private}}/files/backup.pub dest=/var/lib/drbackup/.ssh/authorized_keys owner=drbackup group=drbackup mode=0600 # # deploy the confine-ssh script From 8277847d57a8f5e5361c6230f8f5fcf5b3793223 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 22:31:24 +0000 Subject: [PATCH 170/193] Need mod_ssl here. --- roles/cgit/base/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/cgit/base/tasks/main.yml b/roles/cgit/base/tasks/main.yml index 1ac9239a83..98b1cad369 100644 --- a/roles/cgit/base/tasks/main.yml +++ b/roles/cgit/base/tasks/main.yml @@ -5,6 +5,7 @@ yum: pkg={{item}} state=present with_items: - cgit + - mod_ssl # https://bugzilla.redhat.com/show_bug.cgi?id=1134103 # - lua-md5 From abeeb5c4e4af2af6ff1211e605b62be3232a110a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 22:35:24 +0000 Subject: [PATCH 171/193] This is just Mutex in new apache --- roles/distgit/files/lookaside-upload.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/distgit/files/lookaside-upload.conf b/roles/distgit/files/lookaside-upload.conf index a5948d7398..39c4247618 100644 --- a/roles/distgit/files/lookaside-upload.conf +++ b/roles/distgit/files/lookaside-upload.conf @@ -6,7 +6,7 @@ Listen 443 SSLSessionCache shmcb:/var/cache/mod_ssl/scache(512000) SSLSessionCacheTimeout 300 -SSLMutex default +Mutex default SSLRandomSeed startup file:/dev/urandom 256 SSLRandomSeed connect builtin From be327c2670fba456fd6630c2c46ac1083ffbf79c Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 26 Aug 2014 22:35:56 +0000 Subject: [PATCH 172/193] Theres no longer any rewritelog specific logging in new apache --- roles/distgit/files/redirect.conf | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/distgit/files/redirect.conf b/roles/distgit/files/redirect.conf index dc1b6a1133..20f2bbbe2d 100644 --- a/roles/distgit/files/redirect.conf +++ b/roles/distgit/files/redirect.conf @@ -2,12 +2,6 @@ RedirectMatch permanent ^/$ http://pkgs.fedoraproject.org/cgit/ RewriteEngine on -# rewrite log -# set level to 0 to turn off and speed things up -# when debugging - set to 3 and you can see what the hell is going on -RewriteLogLevel 0 -RewriteLog "/var/log/httpd/git-rewrite.log" - # Use cgit and redirect (some) old gitweb-caching things RewriteRule ^/cgit-data/(.*)$ /cgit-data/$1 [L,PT] RewriteRule ^/cgit/(.*)$ /cgit/$1 [L,PT] From 7dd4ea9832303cd893678d760e8d2e84cb450a9b Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 27 Aug 2014 00:33:42 +0000 Subject: [PATCH 173/193] so this is going to be slow because its recursive directories.. but what ya gonna do? --- ...d.conf.cloud-noc01.cloud.fedoraproject.org | 98 +++++++++---------- roles/tftp_server/tasks/main.yml | 3 + 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org index 5e88145dad..bab07ca2a0 100644 --- a/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.cloud-noc01.cloud.fedoraproject.org @@ -33,145 +33,145 @@ subnet 172.23.0.0 netmask 255.255.255.0 { hardware ethernet fc:2f:40:1b:64:4e; fixed-address 172.23.0.70; option host-name "arm03-packager00-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-packager01-mgmt { hardware ethernet fc:2f:40:08:d7:e2; fixed-address 172.23.0.71; option host-name "arm03-packager01-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-qa00-mgmt { hardware ethernet fc:2f:40:a1:f8:36; fixed-address 172.23.0.72; option host-name "arm03-qa00-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-qa01-mgmt { hardware ethernet fc:2f:40:1b:f6:da; fixed-address 172.23.0.73; option host-name "arm03-qa01-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc04-mgmt { hardware ethernet fc:2f:40:16:c8:3a; fixed-address 172.23.0.74; option host-name "arm03-soc04-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc05-mgmt { hardware ethernet fc:2f:40:9b:1d:da; fixed-address 172.23.0.75; option host-name "arm03-soc05-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc06-mgmt { hardware ethernet fc:2f:40:00:4c:e2; fixed-address 172.23.0.76; option host-name "arm03-soc06-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc07-mgmt { hardware ethernet fc:2f:40:57:cb:b6; fixed-address 172.23.0.77; option host-name "arm03-soc07-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc08-mgmt { hardware ethernet fc:2f:40:12:6a:26; fixed-address 172.23.0.78; option host-name "arm03-soc08-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc09-mgmt { hardware ethernet fc:2f:40:5d:85:8a; fixed-address 172.23.0.79; option host-name "arm03-soc09-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc10-mgmt { hardware ethernet fc:2f:40:a5:5f:42; fixed-address 172.23.0.80; option host-name "arm03-soc10-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc11-mgmt { hardware ethernet fc:2f:40:d2:a1:0e; fixed-address 172.23.0.81; option host-name "arm03-soc11-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc12-mgmt { hardware ethernet fc:2f:40:00:a4:c6; fixed-address 172.23.0.82; option host-name "arm03-soc12-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc13-mgmt { hardware ethernet fc:2f:40:59:27:ba; fixed-address 172.23.0.83; option host-name "arm03-soc13-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc14-mgmt { hardware ethernet fc:2f:40:7b:ab:f6; fixed-address 172.23.0.84; option host-name "arm03-soc14-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc15-mgmt { hardware ethernet fc:2f:40:8a:99:96; fixed-address 172.23.0.85; option host-name "arm03-soc15-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc16-mgmt { hardware ethernet fc:2f:40:ea:ff:16; fixed-address 172.23.0.86; option host-name "arm03-soc16-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc17-mgmt { hardware ethernet fc:2f:40:79:e3:8e; fixed-address 172.23.0.87; option host-name "arm03-soc17-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc18-mgmt { hardware ethernet fc:2f:40:7d:0c:9a; fixed-address 172.23.0.88; option host-name "arm03-soc18-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc19-mgmt { hardware ethernet fc:2f:40:04:29:9a; fixed-address 172.23.0.89; option host-name "arm03-soc19-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc20-mgmt { hardware ethernet fc:2f:40:3c:50:26; fixed-address 172.23.0.90; option host-name "arm03-soc20-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc21-mgmt { hardware ethernet fc:2f:40:cb:4f:66; fixed-address 172.23.0.91; option host-name "arm03-soc21-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc22-mgmt { hardware ethernet fc:2f:40:17:c0:ee; fixed-address 172.23.0.92; option host-name "arm03-soc22-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } host arm03-soc23-mgmt { hardware ethernet fc:2f:40:00:47:3e; fixed-address 172.23.0.93; option host-name "arm03-soc23-mgmt"; - next-server 172.23.0.2; + next-server 172.23.0.17; } } @@ -188,152 +188,152 @@ subnet 209.132.184.0 netmask 255.255.255.0 { hardware ethernet f0:1f:af:e3:5f:0c; fixed-address 209.132.184.9; option host-name "fed-cloud09.cloud.fedoraproject.org"; - next-server 209.132.184.2; + next-server 209.132.184.17; filename "pxelinux.0"; } host arm03-packager00 { hardware ethernet fc:2f:40:1b:64:4c; fixed-address 209.132.184.70; option host-name "arm03-packager00"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-packager01 { hardware ethernet fc:2f:40:08:d7:e0; fixed-address 209.132.184.71; option host-name "arm03-packager01"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-qa00 { hardware ethernet fc:2f:40:a1:f8:34; fixed-address 209.132.184.72; option host-name "arm03-qa00"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-qa01 { hardware ethernet fc:2f:40:1b:f6:d8; fixed-address 209.132.184.73; option host-name "arm03-qa01"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc04 { hardware ethernet fc:2f:40:16:c8:38; fixed-address 209.132.184.74; option host-name "arm03-soc04"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc05 { hardware ethernet fc:2f:40:9b:1d:d8; fixed-address 209.132.184.75; option host-name "arm03-soc05"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc06 { hardware ethernet fc:2f:40:00:4c:e0; fixed-address 209.132.184.76; option host-name "arm03-soc06"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc07 { hardware ethernet fc:2f:40:57:cb:b4; fixed-address 209.132.184.77; option host-name "arm03-soc07"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc08 { hardware ethernet fc:2f:40:12:6a:24; fixed-address 209.132.184.78; option host-name "arm03-soc08"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc09 { hardware ethernet fc:2f:40:5d:85:88; fixed-address 209.132.184.79; option host-name "arm03-soc09"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc10 { hardware ethernet fc:2f:40:a5:5f:40; fixed-address 209.132.184.80; option host-name "arm03-soc10"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc11 { hardware ethernet fc:2f:40:d2:a1:0c; fixed-address 209.132.184.81; option host-name "arm03-soc11"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc12 { hardware ethernet fc:2f:40:00:a4:c4; fixed-address 209.132.184.82; option host-name "arm03-soc12"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc13 { hardware ethernet fc:2f:40:59:27:b8; fixed-address 209.132.184.83; option host-name "arm03-soc13"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc14 { hardware ethernet fc:2f:40:7b:ab:f4; fixed-address 209.132.184.84; option host-name "arm03-soc14"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc15 { hardware ethernet fc:2f:40:8a:99:94; fixed-address 209.132.184.85; option host-name "arm03-soc15"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc16 { hardware ethernet fc:2f:40:ea:ff:14; fixed-address 209.132.184.86; option host-name "arm03-soc16"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc17 { hardware ethernet fc:2f:40:79:e3:8c; fixed-address 209.132.184.87; option host-name "arm03-soc17"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc18 { hardware ethernet fc:2f:40:7d:0c:98; fixed-address 209.132.184.88; option host-name "arm03-soc18"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc19 { hardware ethernet fc:2f:40:04:29:98; fixed-address 209.132.184.89; option host-name "arm03-soc19"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc20 { hardware ethernet fc:2f:40:3c:50:24; fixed-address 209.132.184.90; option host-name "arm03-soc20"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc21 { hardware ethernet fc:2f:40:cb:4f:64; fixed-address 209.132.184.91; option host-name "arm03-soc21"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc22 { hardware ethernet fc:2f:40:17:c0:ec; fixed-address 209.132.184.92; option host-name "arm03-soc22"; - next-server 209.132.184.2; + next-server 209.132.184.17; } host arm03-soc23 { hardware ethernet fc:2f:40:00:47:3c; fixed-address 209.132.184.93; option host-name "arm03-soc23"; - next-server 209.132.184.2; + next-server 209.132.184.17; } } diff --git a/roles/tftp_server/tasks/main.yml b/roles/tftp_server/tasks/main.yml index 6e8be867de..4f600632a5 100644 --- a/roles/tftp_server/tasks/main.yml +++ b/roles/tftp_server/tasks/main.yml @@ -29,3 +29,6 @@ - service - config - tftp_server + +- name: fill up the tftpboot directory + copy: src="{{ bigfiles }}/tftpboot/" dest=/var/lib/tftpboot/ From f875a0fbd2499beeff3acc54d0a1dce65a9e3103 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Wed, 27 Aug 2014 00:55:42 +0000 Subject: [PATCH 174/193] make tftp default file per site --- ...efault.cloud-noc01.cloud.fedoraproject.org | 88 +++++++++++++++++++ ... => default.dhcp01.phx2.fedoraproject.org} | 0 .../default.noc01.phx2.fedoraproject.org | 88 +++++++++++++++++++ roles/tftp_server/tasks/main.yml | 2 +- 4 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 roles/tftp_server/files/default.cloud-noc01.cloud.fedoraproject.org rename roles/tftp_server/files/{default => default.dhcp01.phx2.fedoraproject.org} (100%) create mode 100644 roles/tftp_server/files/default.noc01.phx2.fedoraproject.org diff --git a/roles/tftp_server/files/default.cloud-noc01.cloud.fedoraproject.org b/roles/tftp_server/files/default.cloud-noc01.cloud.fedoraproject.org new file mode 100644 index 0000000000..3d7591a868 --- /dev/null +++ b/roles/tftp_server/files/default.cloud-noc01.cloud.fedoraproject.org @@ -0,0 +1,88 @@ +DEFAULT menu +MENU TITLE Fedora Infrastructure PXE Boot +PROMPT 0 +TIMEOUT 200 +TOTALTIMEOUT 6000 +ONTIMEOUT local + +LABEL local + MENU LABEL (local) + MENU DEFAULT + LOCALBOOT 0 + +LABEL EL-7-nohd + MENU LABEL el7nohd + KERNEL images/RHEL/7/x86_64/vmlinuz + APPEND initrd=images/RHEL/7/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL7-x86_64/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-rhel-7-nohd net.ifnames=0 biosdevname=0 + +LABEL EL-6Rescue + MENU LABEL el6rescue + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ip=dhcp nomodeset rescue + +LABEL EL-6Builder + MENU LABEL ^el6builder + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/rhel-6-builder + +LABEL EL-6virthost + MENU LABEL el6virthost + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-rhel-6-nohd + +LABEL EL-6buildvmhost + MENU LABEL el6buildvmhost + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/buildvmhost-novnc + +LABEL EL6novnc-nohd + MENU LABEL EL-6novnc-nohd + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-rhel-6-novnc + +LABEL EL6cloud + MENU LABEL EL-6cloud + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://209.132.181.6/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-rhel-6-cloud + +LABEL QA-Fed20-x86_64 + MENU LABEL QA-Fedora20-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.124.210/ks/f20-virthost-infra.ks + +LABEL Fed20-x86_64 + MENU LABEL Fedora20-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-f20.cfg vnc vncpassword=vncinstall nomodeset + +LABEL Fed20-x86_64-novnc + MENU LABEL Fedora20-x86_64-novnc + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-f20.cfg nomodeset + +LABEL Fed20-x86_64-buildhw + MENU LABEL Fedora20-x86_64-buildhw + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/buildhw-fedora-20 text net.ifnames=0 + +LABEL Fed20-x86_64-osbuild + MENU LABEL Fedora20-x86_64-osbuild + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-osbuild.cfg text net.ifnames=0 nomodeset + +LABEL Fed20-Rescue-x86_64 + MENU LABEL Fedora-20-Rescue-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND initrd=images/Fedora/20/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp nomodeset rescue + +LABEL Fed19-x86_64 + MENU LABEL Fedora19-x86_64 + KERNEL images/Fedora/19/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/19/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/19/Fedora/x86_64/os/ ip=dhcp ks=http://209.132.181.6/repo/rhel/ks/hardware-f19.cfg vnc vncpassword=vncinstall nomodeset + +LABEL Fed19-Rescue-x86_64 + MENU LABEL Fedora-19-Rescue-x86_64 + KERNEL images/Fedora/19/x86_64/vmlinuz + APPEND initrd=images/Fedora/19/x86_64/initrd.img method=http://209.132.181.6/pub/fedora/linux/releases/19/Fedora/x86_64/os/ ip=dhcp nomodeset rescue + +MENU end diff --git a/roles/tftp_server/files/default b/roles/tftp_server/files/default.dhcp01.phx2.fedoraproject.org similarity index 100% rename from roles/tftp_server/files/default rename to roles/tftp_server/files/default.dhcp01.phx2.fedoraproject.org diff --git a/roles/tftp_server/files/default.noc01.phx2.fedoraproject.org b/roles/tftp_server/files/default.noc01.phx2.fedoraproject.org new file mode 100644 index 0000000000..9fd803ea41 --- /dev/null +++ b/roles/tftp_server/files/default.noc01.phx2.fedoraproject.org @@ -0,0 +1,88 @@ +DEFAULT menu +MENU TITLE Fedora Infrastructure PXE Boot +PROMPT 0 +TIMEOUT 200 +TOTALTIMEOUT 6000 +ONTIMEOUT local + +LABEL local + MENU LABEL (local) + MENU DEFAULT + LOCALBOOT 0 + +LABEL EL-7-nohd + MENU LABEL el7nohd + KERNEL images/RHEL/7/x86_64/vmlinuz + APPEND initrd=images/RHEL/7/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL7-x86_64/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-rhel-7-nohd net.ifnames=0 biosdevname=0 + +LABEL EL-6Rescue + MENU LABEL el6rescue + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ip=dhcp nomodeset rescue + +LABEL EL-6Builder + MENU LABEL ^el6builder + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/rhel-6-builder + +LABEL EL-6virthost + MENU LABEL el6virthost + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-rhel-6-nohd + +LABEL EL-6buildvmhost + MENU LABEL el6buildvmhost + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/buildvmhost-novnc + +LABEL EL6novnc-nohd + MENU LABEL EL-6novnc-nohd + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-rhel-6-novnc + +LABEL EL6cloud + MENU LABEL EL-6cloud + KERNEL images/RHEL/6/x86_64/vmlinuz + APPEND ks initrd=images/RHEL/6/x86_64/initrd.img method=http://10.5.126.23/repo/rhel/RHEL6-x86_64/ ksdevice=eth0 ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-rhel-6-cloud + +LABEL QA-Fed20-x86_64 + MENU LABEL QA-Fedora20-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.124.210/ks/f20-virthost-infra.ks + +LABEL Fed20-x86_64 + MENU LABEL Fedora20-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-f20.cfg vnc vncpassword=vncinstall nomodeset + +LABEL Fed20-x86_64-novnc + MENU LABEL Fedora20-x86_64-novnc + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-f20.cfg nomodeset + +LABEL Fed20-x86_64-buildhw + MENU LABEL Fedora20-x86_64-buildhw + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/buildhw-fedora-20 text net.ifnames=0 + +LABEL Fed20-x86_64-osbuild + MENU LABEL Fedora20-x86_64-osbuild + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-osbuild.cfg text net.ifnames=0 nomodeset + +LABEL Fed20-Rescue-x86_64 + MENU LABEL Fedora-20-Rescue-x86_64 + KERNEL images/Fedora/20/x86_64/vmlinuz + APPEND initrd=images/Fedora/20/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/20/Fedora/x86_64/os/ ip=dhcp nomodeset rescue + +LABEL Fed19-x86_64 + MENU LABEL Fedora19-x86_64 + KERNEL images/Fedora/19/x86_64/vmlinuz + APPEND ks initrd=images/Fedora/19/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/19/Fedora/x86_64/os/ ip=dhcp ks=http://10.5.126.23/repo/rhel/ks/hardware-f19.cfg vnc vncpassword=vncinstall nomodeset + +LABEL Fed19-Rescue-x86_64 + MENU LABEL Fedora-19-Rescue-x86_64 + KERNEL images/Fedora/19/x86_64/vmlinuz + APPEND initrd=images/Fedora/19/x86_64/initrd.img method=http://10.5.126.23/pub/fedora/linux/releases/19/Fedora/x86_64/os/ ip=dhcp nomodeset rescue + +MENU end diff --git a/roles/tftp_server/tasks/main.yml b/roles/tftp_server/tasks/main.yml index 4f600632a5..6024361408 100644 --- a/roles/tftp_server/tasks/main.yml +++ b/roles/tftp_server/tasks/main.yml @@ -18,7 +18,7 @@ - tftp_server - name: setup pxelinux.cfg default file - copy: src=default dest=/var/lib/tftpboot/pxelinux.cfg/default mode=644 + copy: src=default.{{ inventory_hostname }} dest=/var/lib/tftpboot/pxelinux.cfg/default mode=644 tags: - config - tftp_server From 1cc27e1b1b5ca9bb47d2bf6156e4f0d6d4de4d0a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 03:25:56 +0000 Subject: [PATCH 175/193] Adjust docs backend path --- roles/publican_webhost/templates/publican-website.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/publican_webhost/templates/publican-website.cfg b/roles/publican_webhost/templates/publican-website.cfg index 1a2b6747b6..2bf3837184 100644 --- a/roles/publican_webhost/templates/publican-website.cfg +++ b/roles/publican_webhost/templates/publican-website.cfg @@ -1,5 +1,5 @@ db_file: /var/www/html/{{ publican_brand }}.{{ kojitag }}.db -toc_path: /var/www/html/docs/ +toc_path: /var/www/html/docs/fedora/ host: {{ site_host }} title: "site_host" search: '

' From bcd2f6a3f85cb170d2d26e804713fb59260b9010 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 03:28:44 +0000 Subject: [PATCH 176/193] Add sysadmin-fedimg to fedimg hosts --- inventory/group_vars/fedimg | 2 +- inventory/group_vars/fedimg-stg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/fedimg b/inventory/group_vars/fedimg index 1f850148a9..748f24feb2 100644 --- a/inventory/group_vars/fedimg +++ b/inventory/group_vars/fedimg @@ -9,7 +9,7 @@ num_cpus: 2 tcp_ports: [ 3000 ] # TODO, restrict this down to just sysadmin-releng -fas_client_groups: sysadmin-datanommer,sysadmin-releng +fas_client_groups: sysadmin-datanommer,sysadmin-releng,sysadmin-fedimg # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: diff --git a/inventory/group_vars/fedimg-stg b/inventory/group_vars/fedimg-stg index 1f850148a9..748f24feb2 100644 --- a/inventory/group_vars/fedimg-stg +++ b/inventory/group_vars/fedimg-stg @@ -9,7 +9,7 @@ num_cpus: 2 tcp_ports: [ 3000 ] # TODO, restrict this down to just sysadmin-releng -fas_client_groups: sysadmin-datanommer,sysadmin-releng +fas_client_groups: sysadmin-datanommer,sysadmin-releng,sysadmin-fedimg # These are consumed by a task in roles/fedmsg/base/main.yml fedmsg_certs: From 16564bfeb77c7a9b6b96c7f659b90864e7ba0dd8 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 16:09:17 +0000 Subject: [PATCH 177/193] Try and make this not send emails most of the time. --- roles/distgit/files/updatecrl.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/distgit/files/updatecrl.sh b/roles/distgit/files/updatecrl.sh index c7649b3b9d..cf9f934a5a 100644 --- a/roles/distgit/files/updatecrl.sh +++ b/roles/distgit/files/updatecrl.sh @@ -4,7 +4,7 @@ URL=https://admin.fedoraproject.org/ca/crl.pem OLD=/etc/pki/tls/crl.pem NEW=/tmp/crl.pem -wget $URL -O $NEW +wget -q $URL -O $NEW OLDUPDATE=`openssl crl -in $OLD -noout -lastupdate` NEWUPDATE=`openssl crl -in $NEW -noout -lastupdate` @@ -12,5 +12,4 @@ if [ "$OLDUPDATE" != "$NEWUPDATE" ]; then mv $NEW $OLD restorecon $OLD /etc/init.d/httpd graceful - echo "updated to $NEWUPDATE" fi From 36a47b8a5b44f2538a7ec00226c3f5ea90d779c6 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 16:29:07 +0000 Subject: [PATCH 178/193] Adjust some hosts that should/should not freeze --- inventory/group_vars/beaker | 2 ++ inventory/group_vars/kernel-qa | 2 +- inventory/group_vars/qadevel | 2 ++ inventory/group_vars/qadevel-stg | 2 ++ inventory/group_vars/resultsdb-dev | 2 ++ inventory/group_vars/resultsdb-stg | 2 ++ inventory/group_vars/taskotron-dev | 1 + inventory/group_vars/taskotron-dev-clients | 1 + inventory/group_vars/taskotron-stg | 1 + inventory/group_vars/taskotron-stg-clients | 1 + inventory/host_vars/cloud-noc01.phx2.fedoraproject.org | 1 + 11 files changed, 16 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/beaker b/inventory/group_vars/beaker index 59c9a26b0b..1f46194cf6 100644 --- a/inventory/group_vars/beaker +++ b/inventory/group_vars/beaker @@ -11,3 +11,5 @@ udp_ports: [ 69 ] fas_client_groups: sysadmin-qa nrpe_procs_warn: 250 nrpe_procs_crit: 300 + +freezes: false diff --git a/inventory/group_vars/kernel-qa b/inventory/group_vars/kernel-qa index 39e618921d..42f53afa5a 100644 --- a/inventory/group_vars/kernel-qa +++ b/inventory/group_vars/kernel-qa @@ -1,5 +1,5 @@ --- -freezes: true +freezes: false resolvconf: "{{ files }}/resolv.conf/phx2" fas_client_groups: sysadmin-kernel sudoers: "{{ private }}/files/sudo/kernel-qa" diff --git a/inventory/group_vars/qadevel b/inventory/group_vars/qadevel index de0aea8bb9..f962aae6fa 100644 --- a/inventory/group_vars/qadevel +++ b/inventory/group_vars/qadevel @@ -37,3 +37,5 @@ tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] slave_home: /home/buildslave/ slave_dir: /home/buildslave/slave slave_user: buildslave + +freezes: false diff --git a/inventory/group_vars/qadevel-stg b/inventory/group_vars/qadevel-stg index 5f63d78181..caa9ae52ad 100644 --- a/inventory/group_vars/qadevel-stg +++ b/inventory/group_vars/qadevel-stg @@ -32,3 +32,5 @@ master_user: buildmaster external_hostname: qadevel-stg.qa.fedoraproject.org deployment_type: qadevel-stg tcp_ports: [ 80, 443, "{{ buildslave_port }}" ] + +freezes: false diff --git a/inventory/group_vars/resultsdb-dev b/inventory/group_vars/resultsdb-dev index 652351c35f..83d106d3f6 100644 --- a/inventory/group_vars/resultsdb-dev +++ b/inventory/group_vars/resultsdb-dev @@ -26,3 +26,5 @@ resultsdb_fe_endpoint: '/resultsdb' resultsdb_db_name: resultsdb_dev allowed_hosts: - 10.5.124 + +freezes: false diff --git a/inventory/group_vars/resultsdb-stg b/inventory/group_vars/resultsdb-stg index f5ef63e4aa..7032e98d0c 100644 --- a/inventory/group_vars/resultsdb-stg +++ b/inventory/group_vars/resultsdb-stg @@ -27,3 +27,5 @@ resultsdb_fe_endpoint: '/resultsdb' resultsdb_db_name: resultsdb_stg allowed_hosts: - 10.5.124 + +freezes: false diff --git a/inventory/group_vars/taskotron-dev b/inventory/group_vars/taskotron-dev index 76b9b3e112..4b7f93cd4f 100644 --- a/inventory/group_vars/taskotron-dev +++ b/inventory/group_vars/taskotron-dev @@ -34,3 +34,4 @@ fakefedorainfra_db_name: dev_fakefedorainfra fakefedorainfra_endpoint: fakefedorainfra fakefedorainfra_url: https://taskotron-dev.fedoraproject.org/fakefedorainfra taskotron_docs_url: https://docs.qadevel.cloud.fedoraproject.org/libtaskotron/latest/ +freezes: false diff --git a/inventory/group_vars/taskotron-dev-clients b/inventory/group_vars/taskotron-dev-clients index b384c6dab9..679b1785f1 100644 --- a/inventory/group_vars/taskotron-dev-clients +++ b/inventory/group_vars/taskotron-dev-clients @@ -21,3 +21,4 @@ buildslave_public_sshkey_file: dev-buildslave-sshkey/dev_buildslave.pub taskotron_admin_email: taskotron-admin-members@fedoraproject.org sudoers: "{{ private }}/files/sudo/qavirt-sudoers" buildmaster_pubkey: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK4M03mLIZ0Wf9CzoJtUfOV8pcSxYLSsd4zxaFovDIHZGZH3ifg5Ocwut6L6lBalR3iepa/9EuFvgosi90WM3iI=" +freezes: false diff --git a/inventory/group_vars/taskotron-stg b/inventory/group_vars/taskotron-stg index ddfdc1320c..c1822a0e2c 100644 --- a/inventory/group_vars/taskotron-stg +++ b/inventory/group_vars/taskotron-stg @@ -30,3 +30,4 @@ fakefedorainfra_db_name: fakefedorainfra_stg fakefedorainfra_endpoint: fakefedorainfra fakefedorainfra_url: https://taskotron.stg.fedoraproject.org/fakefedorainfra taskotron_docs_url: https://docs.qadevel.cloud.fedoraproject.org/libtaskotron/latest/ +freezes: false diff --git a/inventory/group_vars/taskotron-stg-clients b/inventory/group_vars/taskotron-stg-clients index 3ad236a5b4..e014baedf6 100644 --- a/inventory/group_vars/taskotron-stg-clients +++ b/inventory/group_vars/taskotron-stg-clients @@ -21,3 +21,4 @@ buildslave_public_sshkey_file: stg-buildslave-sshkey/stg_buildslave.pub taskotron_admin_email: taskotron-admin-members@fedoraproject.org sudoers: "{{ private }}/files/sudo/qavirt-sudoers" buildmaster_pubkey: 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJJ4xAImruf8x0ghwxfq0DM6S00pSoEhpI1VZiG2DT14xD+eMubFQcUMpoQ3IBs3eaatlwVr2qjM4EEBfds/1Zs=' +freezes: false diff --git a/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org index dbccb997f9..60705b1c5d 100644 --- a/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org +++ b/inventory/host_vars/cloud-noc01.phx2.fedoraproject.org @@ -8,3 +8,4 @@ fas_client_groups: sysadmin-main dns: 8.8.8.8 eth0_ip: 209.132.184.17 eth1_ip: 172.23.0.17 +freezes: false From 98571dda42592176e208d66c010b7e0659b13a17 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 16:31:57 +0000 Subject: [PATCH 179/193] Some more hosts that should not freeze --- inventory/host_vars/209.132.184.137 | 1 + inventory/host_vars/209.132.184.143 | 1 + inventory/host_vars/209.132.184.146 | 1 + inventory/host_vars/209.132.184.147 | 2 +- inventory/host_vars/209.132.184.148 | 2 +- inventory/host_vars/209.132.184.153 | 1 + inventory/host_vars/209.132.184.157 | 1 + inventory/host_vars/209.132.184.158 | 1 + inventory/host_vars/209.132.184.162 | 1 + inventory/host_vars/209.132.184.165 | 1 + inventory/host_vars/209.132.184.166 | 1 + inventory/host_vars/209.132.184.209 | 1 + 12 files changed, 12 insertions(+), 2 deletions(-) diff --git a/inventory/host_vars/209.132.184.137 b/inventory/host_vars/209.132.184.137 index e59135be6a..ec885f12ad 100644 --- a/inventory/host_vars/209.132.184.137 +++ b/inventory/host_vars/209.132.184.137 @@ -8,3 +8,4 @@ hostbase: jenkins-el7b public_ip: 209.132.184.137 root_auth_users: pingou description: jenkins el7b worker/slave +freezes: false diff --git a/inventory/host_vars/209.132.184.143 b/inventory/host_vars/209.132.184.143 index 49c3c5fcc1..aa0ef6e282 100644 --- a/inventory/host_vars/209.132.184.143 +++ b/inventory/host_vars/209.132.184.143 @@ -9,3 +9,4 @@ public_ip: 209.132.184.143 root_auth_users: duffy kevin description: artboard cloud instance for the fedora art group volumes: ['-d /dev/vdb vol-00000009'] +freezes: false diff --git a/inventory/host_vars/209.132.184.146 b/inventory/host_vars/209.132.184.146 index 277f62e562..07aaceba68 100644 --- a/inventory/host_vars/209.132.184.146 +++ b/inventory/host_vars/209.132.184.146 @@ -9,3 +9,4 @@ public_ip: 209.132.184.146 root_auth_users: lmacken description: cloud instance for developing/testing logstash volumes: ['-d /dev/vdb vol-0000000d'] +freezes: false diff --git a/inventory/host_vars/209.132.184.147 b/inventory/host_vars/209.132.184.147 index 6d2e867791..80b5f027e2 100644 --- a/inventory/host_vars/209.132.184.147 +++ b/inventory/host_vars/209.132.184.147 @@ -9,4 +9,4 @@ public_ip: 209.132.184.147 root_auth_users: pingou description: fedocal dev server volumes: ['-d /dev/vdb vol-00000010'] - +freezes: false diff --git a/inventory/host_vars/209.132.184.148 b/inventory/host_vars/209.132.184.148 index 0c7bc3c3c2..65c26f1421 100644 --- a/inventory/host_vars/209.132.184.148 +++ b/inventory/host_vars/209.132.184.148 @@ -13,4 +13,4 @@ public_ip: 209.132.184.148 # users/groups who should have root ssh access root_auth_users: kushal @sysadmin-main sayanchowdhury description: darkserver dev server - +freezes: false diff --git a/inventory/host_vars/209.132.184.153 b/inventory/host_vars/209.132.184.153 index 2bb37e4ccd..1d4dff3e99 100644 --- a/inventory/host_vars/209.132.184.153 +++ b/inventory/host_vars/209.132.184.153 @@ -9,3 +9,4 @@ public_ip: 209.132.184.153 root_auth_users: pingou puiterwijk description: jenkins cloud master volumes: ['-d /dev/vdb vol-00000011'] +freezes: false diff --git a/inventory/host_vars/209.132.184.157 b/inventory/host_vars/209.132.184.157 index 0c661e4ac3..8a4aee0bbb 100644 --- a/inventory/host_vars/209.132.184.157 +++ b/inventory/host_vars/209.132.184.157 @@ -9,3 +9,4 @@ public_ip: 209.132.184.157 root_auth_users: besser82 description: shogun-ca instance, see ticket 4032, besser82 contact volumes: ['-d /dev/vdb vol-00000026'] +freezes: false diff --git a/inventory/host_vars/209.132.184.158 b/inventory/host_vars/209.132.184.158 index ee690765e4..c871f7dd1d 100644 --- a/inventory/host_vars/209.132.184.158 +++ b/inventory/host_vars/209.132.184.158 @@ -8,3 +8,4 @@ hostbase: jenkins-f19 public_ip: 209.132.184.158 root_auth_users: pingou description: jenkins f19 worker/slave +freezes: false diff --git a/inventory/host_vars/209.132.184.162 b/inventory/host_vars/209.132.184.162 index 0bf91956f2..75cc49137d 100644 --- a/inventory/host_vars/209.132.184.162 +++ b/inventory/host_vars/209.132.184.162 @@ -9,3 +9,4 @@ public_ip: 209.132.184.162 root_auth_users: toshio fchiulli description: cloud instance for developing the next version of the elections app volumes: ['-d /dev/vdb vol-0000000e'] +freezes: false diff --git a/inventory/host_vars/209.132.184.165 b/inventory/host_vars/209.132.184.165 index 4281790b8d..ef9b4bf3d6 100644 --- a/inventory/host_vars/209.132.184.165 +++ b/inventory/host_vars/209.132.184.165 @@ -8,3 +8,4 @@ hostbase: jenkins-el6 public_ip: 209.132.184.165 root_auth_users: pingou description: jenkins el6 worker/slave +freezes: false diff --git a/inventory/host_vars/209.132.184.166 b/inventory/host_vars/209.132.184.166 index 2ac7e7f660..b0f2cff557 100644 --- a/inventory/host_vars/209.132.184.166 +++ b/inventory/host_vars/209.132.184.166 @@ -8,3 +8,4 @@ hostbase: jenkins-f18 public_ip: 209.132.184.166 root_auth_users: pingou description: jenkins f18 worker/slave +freezes: false diff --git a/inventory/host_vars/209.132.184.209 b/inventory/host_vars/209.132.184.209 index 8806cc02c1..7a8dd6ccc7 100644 --- a/inventory/host_vars/209.132.184.209 +++ b/inventory/host_vars/209.132.184.209 @@ -8,3 +8,4 @@ hostbase: jenkins-f20 public_ip: 209.132.184.209 root_auth_users: pingou description: jenkins f20 worker/slave +freezes: false From 0da571b9924a207e355feee489c2bff29a5ec6cc Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 27 Aug 2014 17:05:00 +0000 Subject: [PATCH 180/193] Turn fmn threadpool size down a bit. --- roles/notifs/backend/templates/fmn.consumer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/notifs/backend/templates/fmn.consumer.py b/roles/notifs/backend/templates/fmn.consumer.py index 0799f7d7fc..82b5b06cca 100644 --- a/roles/notifs/backend/templates/fmn.consumer.py +++ b/roles/notifs/backend/templates/fmn.consumer.py @@ -25,8 +25,8 @@ config = { # This sets up four threads to handle incoming messages. At the time of # this commit, all of our fedmsg daemons are running in single-threaded # mode. If we turn it on globally, we should remove this setting. - "moksha.workers_per_consumer": 6, - "moksha.threadpool_size": 15, + "moksha.workers_per_consumer": 4, + "moksha.threadpool_size": 10, # Some configuration for the rule processors "fmn.rules.utils.use_pkgdb2": True, From 4638057c2780c550a105be90aabc828388dddd1d Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 18:38:23 +0000 Subject: [PATCH 181/193] Adjust this for rhel7 --- roles/distgit/files/updatecrl.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/distgit/files/updatecrl.sh b/roles/distgit/files/updatecrl.sh index cf9f934a5a..3994c9b2e2 100644 --- a/roles/distgit/files/updatecrl.sh +++ b/roles/distgit/files/updatecrl.sh @@ -10,6 +10,6 @@ NEWUPDATE=`openssl crl -in $NEW -noout -lastupdate` if [ "$OLDUPDATE" != "$NEWUPDATE" ]; then mv $NEW $OLD - restorecon $OLD - /etc/init.d/httpd graceful + /usr/sbin/restorecon $OLD + /usr/bin/systemctl reload httpd fi From 783ebfa992b32a2f9b16e548eb82c06d4dba182a Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 27 Aug 2014 18:41:45 +0000 Subject: [PATCH 182/193] Fix horrible freshclam default sysconfig file. --- roles/clamav/files/freshclam-sysconfig | 24 ++++++++++++++++++++++++ roles/clamav/tasks/main.yml | 3 +++ 2 files changed, 27 insertions(+) create mode 100644 roles/clamav/files/freshclam-sysconfig diff --git a/roles/clamav/files/freshclam-sysconfig b/roles/clamav/files/freshclam-sysconfig new file mode 100644 index 0000000000..2ca4e92ece --- /dev/null +++ b/roles/clamav/files/freshclam-sysconfig @@ -0,0 +1,24 @@ +## When changing the periodicity of freshclam runs in the crontab, +## this value must be adjusted also. Its value is the timespan between +## two subsequent freshclam runs in minutes. E.g. for the default +## +## | 0 */3 * * * ... +## +## crontab line, the value is 180 (minutes). +# FRESHCLAM_MOD= + +## A predefined value for the delay in seconds. By default, the value is +## calculated by the 'hostid' program. This predefined value guarantees +## constant timespans of 3 hours between two subsequent freshclam runs. +## +## This option accepts two special values: +## 'disabled-warn' ... disables the automatic freshclam update and +## gives out a warning +## 'disabled' ... disables the automatic freshclam silently +# FRESHCLAM_DELAY= + + +### !!!!! REMOVE ME !!!!!! +### REMOVE ME: By default, the freshclam update is disabled to avoid +### REMOVE ME: network access without prior activation +#FRESHCLAM_DELAY=disabled-warn # REMOVE ME diff --git a/roles/clamav/tasks/main.yml b/roles/clamav/tasks/main.yml index bd5c288885..675c02de36 100644 --- a/roles/clamav/tasks/main.yml +++ b/roles/clamav/tasks/main.yml @@ -24,6 +24,9 @@ - name: setup the freshclam configuration copy: src=freshclam.conf dest=/etc/freshclam.conf +- name: enable freshclam by fixing the stupid default sysconfig + copy: src=freshclam-sysconfig dest=/etc/sysconfig/freshclam + - name: setup the freshclam cron job copy: src=freshclam-cron dest=/etc/cron.daily/freshclam From 94263f6b7d926acde9303b8e468112a7d380fc56 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 16:05:05 +0200 Subject: [PATCH 183/193] Fix the authorized_keys file generation for pkgs01.stg for gitolite3 --- inventory/group_vars/pkgs-stg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/pkgs-stg b/inventory/group_vars/pkgs-stg index 654dda260c..943b71df7b 100644 --- a/inventory/group_vars/pkgs-stg +++ b/inventory/group_vars/pkgs-stg @@ -9,8 +9,8 @@ tcp_ports: [80, 443, 9418, 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] fas_client_groups: sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc -fas_client_restricted_app: /usr/bin/gl-auth-command -fas_client_admin_app: /usr/bin/gl-auth-command -s +fas_client_restricted_app: /usr/share/gitolite3/gitolite-shell user +fas_client_admin_app: /usr/share/gitolite3/gitolite-shell admin fas_client_ssh_groups: "@cvs,sysadmin-main,sysadmin-cvs,sysadmin-build,sysadmin-noc" git_group: packager From 961301ae9c8b9026f82dcc5d807d93b59012a427 Mon Sep 17 00:00:00 2001 From: Stephen Smoogen Date: Thu, 28 Aug 2014 14:57:19 +0000 Subject: [PATCH 184/193] dhcpd silently choked on this. fixing --- roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org index 9bb2c2e7b2..ad39fcdc41 100644 --- a/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org +++ b/roles/dhcp_server/files/dhcpd.conf.noc01.phx2.fedoraproject.org @@ -1452,7 +1452,7 @@ shared-network qa { filename "yaboot"; } - host ppc-comm04.qa.fedoraproject.org{ + host ppc-comm04.qa.fedoraproject.org { hardware ethernet 5c:f3:fc:89:bd:c0; fixed-address 10.5.124.221; next-server ppc-builder7.qa.fedoraproject.org; From e2a513f27054307c4171ab487a5823fccb9533af Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 17:35:00 +0200 Subject: [PATCH 185/193] Remove the pkgdb2branch.py script --- roles/distgit/files/pkgdb2branch.py | 362 ---------------------------- 1 file changed, 362 deletions(-) delete mode 100644 roles/distgit/files/pkgdb2branch.py diff --git a/roles/distgit/files/pkgdb2branch.py b/roles/distgit/files/pkgdb2branch.py deleted file mode 100644 index 507fdd120b..0000000000 --- a/roles/distgit/files/pkgdb2branch.py +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/python -t -# Author: Toshio Kuratomi -# Copyright: 2007-2008 Red Hat Software -# License: GPLv2 -# This needs a proper license and copyright here -__version__ = '0.3' - -import sys -import os -import optparse - -import subprocess - -import fedmsg - -# Do some off-the-bat configuration of fedmsg. -# 1) since this is a one-off script and not a daemon, it needs to connect to -# the fedmsg-relay process running on another node (or noone will hear it) -# 2) its going to use the 'shell' certificate which only 'sysadmin' has read -# access to. Contrast that with the 'scm' certificate which everyone in -# the 'packager' group has access to. -config = fedmsg.config.load_config([], None) -config['active'] = True -config['endpoints']['relay_inbound'] = config['relay_inbound'] -fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) - -from fedora.client import FedoraServiceError -from pkgdb2client import PkgDB - -GITDIR='/srv/git/rpms' -BASEURL = os.environ.get('PACKAGEDBURL') or 'https://admin.fedoraproject.org/pkgdb/' -MKBRANCH='/usr/local/bin/mkbranch' -SETUP_PACKAGE='/usr/local/bin/setup_git_package' -BRANCHES = {'el4': 'master', 'el5': 'master', 'el6': 'master', 'epel7': 'f19', - 'olpc2': 'f7', - 'olpc3': 'f11', - 'master': None, - 'fc6': 'master', - 'f7': 'master', - 'f8': 'master', - 'f9': 'master', - 'f10': 'master', - 'f11': 'master', - 'f12': 'master', - 'f13': 'master', 'f14': 'master', - 'f15': 'master', 'f16': 'master', - 'f17': 'master', 'f18': 'master', - 'f19': 'master', 'f20': 'master' - } - -# The branch names we get out of pkgdb have to be translated to git -GITBRANCHES = {'el4': 'el4', 'el5': 'el5', 'el6': 'el6', 'epel7': 'epel7', - 'OLPC-2': 'olpc2', - 'FC-6': 'fc6', 'F-7': 'f7', 'F-8': 'f8', 'F-9': 'f9', - 'F-10': 'f10', 'OLPC-3': 'olpc3', - 'F-11': 'f11', 'F-12': 'f12', 'F-13': 'f13', 'f14': 'f14', 'f15': 'f15', 'f16': 'f16', 'f17': 'f17', - 'f18': 'f18', 'f19': 'f19', 'f20': 'f20', - 'devel': 'master'} - -# The branch options we get from the CLI have to be translated to pkgdb -BRANCHBYGIT = dict([(v, k) for (k, v) in GITBRANCHES.iteritems()]) - -class InternalError(Exception): - pass - -class PackageDBError(InternalError): - pass - -class ProcessError(InternalError): - pass - -class ArgumentsError(InternalError): - pass - -class InvalidBranchError(PackageDBError): - pass - -class PackageDBClient(PkgDB): - def __init__(self, baseURL): - '''Initialize the connection. - - Args: - :baseURL: URL from which the packageDB is accessed - ''' - # We're only performing read operations so we don't need a username - super(PackageDBClient, self).__init__(baseURL) - - def get_package_branches(self, pkgname): - '''Return the branches to which a package belongs. - - Args: - :pkgname: The package to retrieve branch information about - ''' - - data = self.get_package(pkgname) - return map(lambda x: x['collection']['branchname'], data['packages']) - - def get_package_list(self, branchName): - '''Retrieve all the packages in a specific branch. - - Args: - :branchName: to return the packages for - ''' - pkgs = map(lambda l: l['name'], self.get_packages('*', branchName, page=0)['packages']) - return pkgs - -class Brancher(object): - ''' Make branches in the GIT Repository.''' - - def __init__(self, pkgdburl, cache, verbose): - # Connect to the package database - self.verbose = verbose - self.client = PackageDBClient(BASEURL) - - def _invoke(self, program, args): - '''Run a command and raise an exception if an error occurred. - - Args: - :program: The program to invoke - :args: List of arguments to pass to the program - - raises ProcessError if there's a problem. - ''' - cmdLine = [program] - cmdLine.extend(args) - print ' '.join(cmdLine) - - stdoutfd = subprocess.PIPE - if self.verbose: - program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) - else: - program = subprocess.Popen(cmdLine, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - retCode = program.wait() - if retCode != 0: - e = ProcessError() - e.returnCode = retCode - e.cmd = ' '.join(cmdLine) - if self.verbose: - output = program.stdout.read() - e.message = 'Error, "%s" returned %s: %s' % (e.cmd, e.returnCode, output) - else: - e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) - raise e - - def _create_branch(self, pkgname, branch): - '''Create a specific branch for a package. - - Args: - :pkgname: Name of the package to branch - :branch: Name of the branch to create - - raises InvalidBranchError if a branchname is unknown. - - Will ignore a branch which is EOL. - ''' - try: - branchFrom = '%s/master' % BRANCHES[branch] - except KeyError: - raise InvalidBranchError( - 'PackageDB returned an invalid branch %s for %s' % - (branch, pkgname)) - - # Add the master to the branch - # No longer add this after the new branching setup. - #branch = '%s/master' % branch - # If branchFrom is None, this is an EOL release - # If the directory already exists, no need to invoke mkbranch - if branchFrom: - # Fall back to branching from master. - frombranchpath = os.path.join(GITDIR, '%s.git' % pkgname, - 'refs/heads', branchFrom) - if not os.path.exists(frombranchpath): - branchFrom = 'master' - - branchpath = os.path.join(GITDIR, '%s.git' % pkgname, - 'refs/heads', branch) - if not os.path.exists(branchpath): - try: - self._invoke(MKBRANCH, ['-s', branchFrom, branch, pkgname]) - except ProcessError, e: - if e.returnCode == 255: - # This is a warning, not an error - return - raise - finally: - fedmsg.publish( - topic='branch', - modname='git', - msg=dict( - agent=os.getlogin(), - name=pkgname, - branch=branch, - ), - ) - - def branch_package(self, pkgname): - '''Create all the branches that are listed in the pkgdb for a package. - - Args: - :pkgname: The package to create branches for - - Note: this will ignore branches which are EOL. - - raises PackageDBError if the package is not present in the Package - Database. - ''' - # Retrieve branch information - try: - branches = self.client.get_package_branches(pkgname) - except FedoraServiceError, e: - raise PackageDBError( - 'Unable to retrieve information about %s: %s' % - (pkgname, str(e))) - - # Create the devel branch if necessary - if not os.path.exists(os.path.join(GITDIR, - '%s.git' % pkgname)): - self._invoke(SETUP_PACKAGE, [pkgname]) - # Create all the required branches for the package - # Use the translated branch name until pkgdb falls inline - for branch in branches: - if branch == 'devel': - continue - if not branch in GITBRANCHES.keys(): - print 'Skipping unknown branch %s' % branch - continue - self._create_branch(pkgname, GITBRANCHES[branch]) - - def mass_branch(self, branchName): - '''Make sure all packages listed for a specific branch in the PackageDB - have a CVS branch. - - Args: - :branchName: The branch to ensure. - ''' - fedmsg.publish( - topic='mass_branch.start', - modname='git', - msg=dict(agent=os.getlogin()), - ) - # Retrieve all the packages in this branch - pkglist = self.client.get_package_list(branchName) - pkglist.sort() - for pkg in pkglist: - # Create a branch for this release for each of them - # Use the translated branch name until pkgdb falls inline - self._create_branch(pkg, GITBRANCHES[branchName]) - - fedmsg.publish( - topic='mass_branch.complete', - modname='git', - msg=dict(agent=os.getlogin()), - ) - -def parse_commands(): - parser = optparse.OptionParser(version=__version__, usage='''pkgdb2branch.py [options] PACKAGENAME [packagename, ...] [-] - pkgdb2branch.py [options] --branchfor BRANCH - -pkgdb2branch reads package information from the packagedb and creates branches -on the git server based on what branches are listed there. pkgdb2branch can -read the list of packages from stdin if you specify '-' as an argument. - -pkgdb2branch has two modes of operation. In the first mode, you specify which -packages you want to branch. This mode is more efficient for a small number -of packages. - -In the second mode, pkgdb2branch will find every package lacking a BRANCH and -will create one if the pkgdb says it's needed. This mode is very efficient for -mass branching. This implies --cache-branches. - -For those with a moderate number of packages, using a list of packages and ---cache-branches may be fastest.''') - parser.add_option('-b', '--branch-for', - dest='branchFor', - action='store', - help='Make sure all the packages have been branched for BRANCHFOR. Implies -c.') - parser.add_option('-c', '--cache-branches', - dest='enableCache', - action='store_true', - help='Download a complete cache of packages') - parser.add_option('--verbose', - dest='verbose', - action='store_true', - help='Enable verbose output') - (opts, args) = parser.parse_args() - - if opts.branchFor: - if args: - raise ArgumentsError('Cannot specify packages with --branchFor') - opts.enableCache = True - - if '-' in args: - opts.fromStdin = True - del (args[args.index('-')]) - else: - opts.fromStdin = False - - if not (args or opts.fromStdin or opts.branchFor): - raise ArgumentsError('You must list packages to operate on') - - return opts, args - -if __name__ == '__main__': - try: - options, packages = parse_commands() - except ArgumentsError, e: - print e - sys.exit(1) - - branchedPackages, unbranchedPackages = [], [] - brancher = Brancher(BASEURL, options.enableCache, options.verbose) - fedmsg.publish( - topic='pkgdb2branch.start', - modname='git', - msg=dict(agent=os.getlogin()), - ) - - if options.branchFor: - try: - unbranchedPackages = \ - brancher.mass_branch(BRANCHBYGIT[options.branchFor]) - except PackageDBError, e: - print 'Unable contact the PackageDB. Error: %s' % str(e) - sys.exit(1) - else: - # Process packages specified on the cmdline - for pkgname in packages: - try: - brancher.branch_package(pkgname) - branchedPackages.append(pkgname) - except InternalError, e: - print str(e) - unbranchedPackages.append(pkgname) - - # Process packages from stdin - if options.fromStdin: - for pkgname in sys.stdin: - pkgname = pkgname.strip() - try: - brancher.branch_package(pkgname) - branchedPackages.append(pkgname) - except InternalError, e: - print str(e) - unbranchedPackages.append(pkgname) - - fedmsg.publish( - topic='pkgdb2branch.complete', - modname='git', - msg=dict( - agent=os.getlogin(), - branchedPackages=branchedPackages, - unbranchedPackages=unbranchedPackages, - ), - ) - - if unbranchedPackages: - print 'The following packages were unbranched:' - print '\n'.join(unbranchedPackages) - sys.exit(100) - - sys.exit(0) From 49e0ca88eada0a5130540f27a94d2d62752ae8cb Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 17:35:18 +0200 Subject: [PATCH 186/193] Add the pkgdb_sync_git_branches.py script --- .../distgit/files/pkgdb_sync_git_branches.py | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 roles/distgit/files/pkgdb_sync_git_branches.py diff --git a/roles/distgit/files/pkgdb_sync_git_branches.py b/roles/distgit/files/pkgdb_sync_git_branches.py new file mode 100644 index 0000000000..42833c2881 --- /dev/null +++ b/roles/distgit/files/pkgdb_sync_git_branches.py @@ -0,0 +1,246 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- + +""" +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +This script is able to query pkgdb and retrieve for all packages which active +branches should be there, browse all the git repos and find out which active +branches are missing. + +It even goes one step further but actually adjusting the git repo by adding +the missing branches (or even the missing repo) + +""" + +import os +import subprocess + +import requests + +import fedmsg + +# Do some off-the-bat configuration of fedmsg. +# 1) since this is a one-off script and not a daemon, it needs to connect +# to the fedmsg-relay process running on another node (or noone will +# hear it) +# 2) its going to use the 'shell' certificate which only 'sysadmin' has +# read access to. Contrast that with the 'scm' certificate which +# everyone in the 'packager' group has access to. + +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) + +PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' +GIT_FOLDER = '/srv/git/rpms/' +MKBRANCH = '/usr/local/bin/mkbranch' +SETUP_PACKAGE = '/usr/local/bin/setup_git_package' + +BRANCHES_FROM = { + 'epel7': 'f19', + 'olpc2': 'f7', + 'olpc3': 'f11', + 'master': None, +} +VERBOSE = False + + +class InternalError(Exception): + pass + + +class ProcessError(InternalError): + pass + + +def _invoke(program, args): + '''Run a command and raise an exception if an error occurred. + + :arg program: The program to invoke + :args: List of arguments to pass to the program + + raises ProcessError if there's a problem. + ''' + cmdLine = [program] + cmdLine.extend(args) + if VERBOSE: + print ' '.join(cmdLine) + + if VERBOSE: + program = subprocess.Popen(cmdLine, stderr=subprocess.STDOUT) + else: + program = subprocess.Popen( + cmdLine, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + retCode = program.wait() + if retCode != 0: + e = ProcessError() + e.returnCode = retCode + e.cmd = ' '.join(cmdLine) + if VERBOSE: + output = program.stdout.read() + e.message = 'Error, "%s" returned %s: %s' % ( + e.cmd, e.returnCode, output) + else: + e.message = 'Error, "%s" returned %s' % (e.cmd, e.returnCode) + raise e + + +def _create_branch(pkgname, branch): + '''Create a specific branch for a package. + + :arg pkgname: Name of the package to branch + :arg branch: Name of the branch to create + + ''' + + if branch in BRANCHES_FROM: + branch_from = BRANCHES_FROM[branch] + else: + branch_from = 'master' + + # Fall back to branching from master. + frombranchpath = os.path.join( + GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch_from) + if not os.path.exists(frombranchpath): + branch_from = 'master' + + branchpath = os.path.join( + GIT_FOLDER, '%s.git' % pkgname, 'refs/heads', branch) + if not os.path.exists(branchpath): + try: + _invoke(MKBRANCH, ['-s', branch_from, branch, pkgname]) + except ProcessError, e: + if e.returnCode == 255: + # This is a warning, not an error + return + raise + finally: + fedmsg.publish( + topic='branch', + modname='git', + msg=dict( + agent=os.getlogin(), + name=pkgname, + branch=branch, + ), + ) + elif VERBOSE: + print 'Was asked to create branch %s of package %s, but it '\ + 'already exists' % (pkgname, branch) + + +def pkgdb_pkg_branch(): + """ Queries pkgdb information about VCS and return a dictionnary of + which branches are available for which packages. + + :return: a dict[pkg_name] = [pkg_branches] + :rtype: dict + """ + url = '%s/api/vcs' % PKGDB_URL + req = requests.get(url, params={'format': 'json'}) + data = req.json() + + output = {} + for pkg in data['packageAcls']: + if pkg in output: + if VERBOSE: + print 'Strange package: %s, it is present twice in the ' \ + 'pkgdb output' % pkg + output[pkg].updated(data['packageAcls'][pkg].keys()) + else: + output[pkg] = set(data['packageAcls'][pkg].keys()) + + return output + + +def get_git_branch(pkg): + """ For the specified package name, check the local git and return the + list of branches found. + """ + git_folder = os.path.join(GIT_FOLDER, '%s.git' % pkg) + if not os.path.exists(git_folder): + print 'Could not find %s' % git_folder + return set() + + head_folder = os.path.join(git_folder, 'refs', 'heads') + return set(os.listdir(head_folder)) + + +def branch_package(pkgname, branches): + '''Create all the branches that are listed in the pkgdb for a package. + + :arg pkgname: The package to create branches for + :arg branches: The branches to creates + + ''' + if VERBOSE: + print 'Fixing package %s for branches %s' % (pkgname, branches) + + # Create the devel branch if necessary + if not os.path.exists( + os.path.join(GIT_FOLDER, '%s.git' % pkgname)): + _invoke(SETUP_PACKAGE, [pkgname]) + + # Create all the required branches for the package + # Use the translated branch name until pkgdb falls inline + for branch in branches: + _create_branch(pkgname, branch) + + +def main(): + """ For each package found via pkgdb, check the local git for its + branches and fix inconsistencies. + """ + + local_pkgs = set(os.listdir(GIT_FOLDER)) + local_pkgs = set([it.replace('.git', '') for it in local_pkgs]) + + pkgdb_info = pkgdb_pkg_branch() + + pkgdb_pkgs = set(pkgdb_info.keys()) + + ## Commented out as we keep the git of retired packages while they won't + ## show up in the information retrieved from pkgdb. + + #if (local_pkgs - pkgdb_pkgs): + #print 'Some packages are present locally but not on pkgdb:' + #print ', '.join(sorted(local_pkgs - pkgdb_pkgs)) + + if (pkgdb_pkgs - local_pkgs): + print 'Some packages are present in pkgdb but not locally:' + print ', '.join(sorted(pkgdb_pkgs - local_pkgs)) + + tofix = set() + for pkg in sorted(pkgdb_info): + pkgdb_branches = pkgdb_info[pkg] + git_branches = get_git_branch(pkg) + diff = (pkgdb_branches - git_branches) + if diff: + print '%s missing: %s' % (pkg, ','.join(sorted(diff))) + tofix.add(pkg) + branch_package(pkg, diff) + + if tofix: + print 'Packages fixed (%s): %s' % ( + len(tofix), ', '.join(sorted(tofix))) + + +if __name__ == '__main__': + import sys + sys.exit(main()) From 5a5129d5a9cf44638a9f26eb1e218d2fdc76ef84 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 17:36:10 +0200 Subject: [PATCH 187/193] Replace install pkgdb2branch by installing pkgdb_sync_git_branches.py --- roles/distgit/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 94e487ef3c..a9c9a4f3af 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -47,7 +47,7 @@ - setup_git_package - mkbranch - pkgdb2-clone - - pkgdb2branch.py + - pkgdb_sync_git_branches.py - process-git-requests - name: install the Dist Git-related httpd config From 34799ff1e954ab2bbc3bcba319650b9662f7ba7f Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 17:39:32 +0200 Subject: [PATCH 188/193] Add and install a daily cron job running pkgdb_sync_git_branches --- roles/distgit/files/pkgdb_sync_git_branches.cron | 1 + roles/distgit/tasks/main.yml | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 roles/distgit/files/pkgdb_sync_git_branches.cron diff --git a/roles/distgit/files/pkgdb_sync_git_branches.cron b/roles/distgit/files/pkgdb_sync_git_branches.cron new file mode 100644 index 0000000000..01946bd127 --- /dev/null +++ b/roles/distgit/files/pkgdb_sync_git_branches.cron @@ -0,0 +1 @@ +00 45 * * * root /usr/local/bin/pkgdb_sync_git_branches.py diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index a9c9a4f3af..80f8cdef4e 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -55,6 +55,14 @@ notify: - restart httpd +- name: pkgdb_sync_git_branches cron job + copy: src=pkgdb_sync_git_branches.cron + dest=/etc/cron.d/pkgdb_sync_git_branches + owner=root mode=644 + tags: + - config + - distgit + # -- Gitolite -------------------------------------------- # This is the permission management for package maintainers, using Gitolite. From b924eae2ce5f110f7a46b96e3bf8b2fd69b0ed6c Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 17:58:53 +0200 Subject: [PATCH 189/193] Rename and adjust the playbook to run pkgdb_sync_git_branches.py --- playbooks/run_pkgdb_sync_git.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 playbooks/run_pkgdb_sync_git.yml diff --git a/playbooks/run_pkgdb_sync_git.yml b/playbooks/run_pkgdb_sync_git.yml new file mode 100644 index 0000000000..02e985d7e5 --- /dev/null +++ b/playbooks/run_pkgdb_sync_git.yml @@ -0,0 +1,16 @@ +# Run `pkgdb2branch` on +# +#To update from testing, adjust as follow: +# --extra-vars="package='pkg1;pkg2;pkg3'" + + +- name: run pkgdb_sync_git_branches.py + hosts: pkgs01.phx2.fedoraproject.org + user: root + serial: 25 + gather_facts: False + + tasks: + - name: call pkgdb_sync_git_branches.py + command: /usr/local/bin/pkgdb_sync_git_branches.py + From d494be610cdec2d8a9243fcc73a49e5876863298 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 18:00:00 +0200 Subject: [PATCH 190/193] Rename failed (pebkac), so remove old playbook --- playbooks/run_pkgdb2branch.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 playbooks/run_pkgdb2branch.yml diff --git a/playbooks/run_pkgdb2branch.yml b/playbooks/run_pkgdb2branch.yml deleted file mode 100644 index bdb20e01fd..0000000000 --- a/playbooks/run_pkgdb2branch.yml +++ /dev/null @@ -1,16 +0,0 @@ -# Run `pkgdb2branch` on -# -#To update from testing, adjust as follow: -# --extra-vars="package='pkg1;pkg2;pkg3'" - - -- name: run pkgdb2branch for the specified packages - hosts: pkgs01.phx2.fedoraproject.org - user: root - serial: 25 - gather_facts: False - - tasks: - - name: call pkgdb2branch.py on {{ package }} - command: /usr/local/bin/pkgdb2branch.py {{ package | join(" ") }} - From 64f77e40bdd604ef4587f26a803aae705cace4ff Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 18:17:58 +0200 Subject: [PATCH 191/193] Move the pkgdb_sync_git_branches.py from a file to a template --- roles/distgit/{files => templates}/pkgdb_sync_git_branches.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/distgit/{files => templates}/pkgdb_sync_git_branches.py (100%) diff --git a/roles/distgit/files/pkgdb_sync_git_branches.py b/roles/distgit/templates/pkgdb_sync_git_branches.py similarity index 100% rename from roles/distgit/files/pkgdb_sync_git_branches.py rename to roles/distgit/templates/pkgdb_sync_git_branches.py From d4dca172ab93c1c01095777c3f8f1614d4ab3fe7 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 18:21:42 +0200 Subject: [PATCH 192/193] Use pkgdb.stg in stg --- roles/distgit/templates/pkgdb_sync_git_branches.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/distgit/templates/pkgdb_sync_git_branches.py b/roles/distgit/templates/pkgdb_sync_git_branches.py index 42833c2881..99c70f5ef2 100644 --- a/roles/distgit/templates/pkgdb_sync_git_branches.py +++ b/roles/distgit/templates/pkgdb_sync_git_branches.py @@ -46,7 +46,12 @@ config['active'] = True config['endpoints']['relay_inbound'] = config['relay_inbound'] fedmsg.init(name='relay_inbound', cert_prefix='shell', **config) +{% if env == 'staging' %} +PKGDB_URL = 'https://admin.stg.fedoraproject.org/pkgdb' +{% else %} PKGDB_URL = 'https://admin.fedoraproject.org/pkgdb' +{% endif %} + GIT_FOLDER = '/srv/git/rpms/' MKBRANCH = '/usr/local/bin/mkbranch' SETUP_PACKAGE = '/usr/local/bin/setup_git_package' From 593b37b4299a34250c52f33c3e0322c3132a9daa Mon Sep 17 00:00:00 2001 From: Pierre-Yves Chibon Date: Thu, 28 Aug 2014 18:22:55 +0200 Subject: [PATCH 193/193] Adjust role to install pkgdb_sync_git_branches.py as a template --- roles/distgit/tasks/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/distgit/tasks/main.yml b/roles/distgit/tasks/main.yml index 80f8cdef4e..d890b73242 100644 --- a/roles/distgit/tasks/main.yml +++ b/roles/distgit/tasks/main.yml @@ -47,7 +47,6 @@ - setup_git_package - mkbranch - pkgdb2-clone - - pkgdb_sync_git_branches.py - process-git-requests - name: install the Dist Git-related httpd config @@ -55,6 +54,14 @@ notify: - restart httpd + name: install the pkgdb_sync_git_branches.py scripts + template: src={{item}} dest=/usr/local/bin/{{item}} owner=root group=root mode=0755 + with_items: + - pkgdb_sync_git_branches.py + tags: + - config + - distgit + - name: pkgdb_sync_git_branches cron job copy: src=pkgdb_sync_git_branches.cron dest=/etc/cron.d/pkgdb_sync_git_branches