A start at pdc config. still need to work out saml2/mellon stuff.

playbooks/groups/pdc-web.yml

@@ -0,0 +1,5 @@
+# create a new notifs-web server
+# NOTE: should be used with --limit most of the time
+# NOTE: make sure there is room/space for this server on the vmhost
+# NOTE: most of these vars_path come from group_vars/notifs-web* or from hostvars
+

playbooks/groups/pdc.yml

@@ -0,0 +1,65 @@
+# PDC servers (both frontend and backend)
+
+- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pdc-web:pdc-web-stg:pdc-backend:pdc-backend-stg"
+
+- name: dole out the generic configuration
+  user: root
+  gather_facts: True
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+  roles:
+  - base
+  - rkhunter
+  - nagios_client
+  - collectd/base
+  - hosts
+  - fas_client
+  - sudo
+  - fedmsg/base
+
+  tasks:
+  - include: "{{ tasks }}/yumrepos.yml"
+  - include: "{{ tasks }}/2fa_client.yml"
+  - include: "{{ tasks }}/motd.yml"
+
+- name: stuff for the web nodes
+  hosts: notifs-web;notifs-web-stg
+  user: root
+  gather_facts: True
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+  roles:
+  - role: openvpn/client
+    when: env != "staging"
+  - apache
+  - pdc/frontend
+
+  tasks:
+  - include: "{{ tasks }}/mod_wsgi.yml"
+
+- name: stuff just for the backend nodes
+  hosts: pdc-backend;pdc-backend-stg
+  user: root
+  gather_facts: True
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "/srv/private/ansible/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+  roles:
+  - fedmsg/hub
+  - pdc/backend
+  - role: collectd/fedmsg-service
+    process: fedmsg-hub

playbooks/hosts/pdc.fedorainfracloud.org.yml

@@ -31,7 +31,7 @@
   roles:
   - base
   - postgresql_server
-  - pdc
+  - pdc/frontend
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"

roles/pdc/backend/files/hotfix/rpmdev-bumpspec

@@ -0,0 +1,336 @@
+#!/usr/bin/python -tt
+# -*- mode: Python; indent-tabs-mode: nil; coding: utf-8 -*-
+#
+# Copyright (c) 2005-2014 Fedora Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+import re
+import subprocess
+import sys
+import textwrap
+import time
+from optparse import OptionParser
+
+
+__version__ = "1.0.13"
+
+
+class BumpSpecError(Exception):
+    pass
+
+
+class SpecFile:
+    def __init__(self, filename, verbose=False, string=None):
+        self.verbose = verbose
+        self.string = string
+
+        self.filename = filename
+        f = None
+        try:
+            f = open(filename, "r")
+            self.lines = f.readlines()
+        finally:
+            f and f.close()
+
+    # supported release value macro definitions
+    _macro_bump_patterns = (
+        re.compile(r"^%(?:define|global)\s+(?i)release\s+(\d+.*)"),
+        re.compile(r"^%(?:define|global)\s+(?i)baserelease\s+(\d+.*)"),
+        )
+    # normal "Release:" tag lines
+    _tag_bump_patterns = (
+        re.compile(r"^Release\s*:\s*(\d+.*)", re.I),
+        re.compile(r"^Release\s*:\s+%release_func\s+(\d+.*)", re.I),
+        )
+    # lines we don't want to mess with
+    _skip_patterns = (
+        re.compile(r"\$Revision:"),
+        )
+
+    def bumpRelease(self):
+        # remember whether we've bumped a macro definition
+        bumped_macro = False
+        # count how many times/lines we've bumped
+        bumped = 0
+
+        for i in range(len(self.lines)):
+            # If we've bumped a macro, we assume this is enough for
+            # the rest of the spec file, so we don't bump a macro and
+            # a corresponding Release tag. The macro may or may not be
+            # used for the definition of one or more Release tags.
+            # Macro-madness makes that hard to check for.
+            if bumped_macro:
+                break
+
+            skipped = False
+            for pattern in SpecFile._skip_patterns:
+                if pattern.search(self.lines[i]):
+                    skipped = True
+                    break
+            if skipped:
+                continue
+
+            for pattern in SpecFile._macro_bump_patterns:
+                (self.lines[i], n) = \
+                    pattern.subn(self.increase, self.lines[i], 1)
+                if n:  # this pattern has lead to a change
+                    bumped += 1
+                    bumped_macro = True
+                    break
+            else:  # no pattern matched
+                for pattern in SpecFile._tag_bump_patterns:
+                    (self.lines[i], n) = \
+                        pattern.subn(self.increase, self.lines[i], 1)
+                    if n:  # this pattern has lead to a change
+                        bumped += 1
+                        break
+                else:  # no pattern matched at all
+                    # Bump ^Release: ... line least-significant.
+                    if self.lines[i].lower().startswith('release:'):
+                        old = self.lines[i][len('Release:'):].rstrip()
+                        new = self.increaseFallback(old)
+                        if self.verbose:
+                            self.debugdiff(old, new)
+                        if old != new:
+                            self.lines[i] = self.lines[i].replace(old, new)
+                            bumped += 1
+
+        if bumped:
+            return
+        if self.verbose:
+            sys.stderr.write('ERROR: No release value matched: %s\n' %
+                             self.filename)
+        sys.exit(1)
+
+    def newVersion(self, vr):
+        rpos = vr.find('-')
+        if rpos >= 0:  # set custom Release value
+            r = vr[rpos+1:]
+            v = vr[:rpos]
+        else:
+            r = "1%{?dist}"
+            v = vr
+        for i in range(len(self.lines)):
+            if self.lines[i].lower().startswith('version:'):
+                self.lines[i] = re.sub(
+                    r'[^: \t]*$', v, self.lines[i].rstrip()) + '\n'
+            elif self.lines[i].lower().startswith('release:'):
+                self.lines[i] = re.sub(
+                    r'[^: \t]*$', r, self.lines[i].rstrip()) + '\n'
+
+    _changelog_pattern = re.compile(r"^%changelog(\s|$)", re.I)
+
+    def addChangelogEntry(self, evr, entry, email):
+        for i in range(len(self.lines)):
+            if SpecFile._changelog_pattern.match(self.lines[i]):
+                if len(evr):
+                    evrstring = ' - %s' % evr
+                else:
+                    evrstring = ''
+                date = time.strftime("%a %b %d %Y", time.gmtime())
+                newchangelogentry = "* %s %s%s\n%s\n\n" % \
+                    (date, email, evrstring, entry)
+                self.lines[i] += newchangelogentry
+                return
+
+    _main_pre_pattern = re.compile(r'^0\.(?P<rel>\d+)(?P<post>.*)')
+    _main_pattern = re.compile(r'^(?P<rel>\d+)(?P<post>.*)')
+
+    def increaseMain(self, release):
+        if release.startswith('0.'):
+            relre = SpecFile._main_pre_pattern
+            pre = True
+        else:
+            relre = SpecFile._main_pattern
+            pre = False
+        relmatch = relre.search(release)
+        if not relmatch:  # pattern match failed
+            raise BumpSpecError
+        value = str(int(relmatch.group('rel')) + 1)
+        post = relmatch.group('post')
+
+        new = value + post
+        if not pre:
+            if post.find('rc') >= 0:
+                sys.stderr.write(
+                    'WARNING: Bad pre-release versioning scheme: %s\n' %
+                    self.filename)
+                raise BumpSpecError
+        else:
+            new = '0.' + new
+        return new
+
+    _jpp_pattern = \
+        re.compile(r'(?P<prefix>.*)(?P<rel>\d+)(?P<jpp>jpp\.)(?P<post>.*)')
+
+    def increaseJPP(self, release):
+        """Fedora jpackage release versioning scheme"""
+
+        relmatch = SpecFile._jpp_pattern.search(release)
+        if not relmatch:  # pattern match failed
+            sys.stderr.write(
+                'WARNING: Bad Fedora jpackage release versioning scheme: %s\n'
+                % self.filename)
+            raise BumpSpecError
+
+        prefix = relmatch.group('prefix')
+        value = int(relmatch.group('rel'))
+        jpp = relmatch.group('jpp')
+        post = relmatch.group('post')
+
+        newpost = self.increaseMain(post)
+        new = prefix+str(value)+jpp+newpost
+        return new
+
+    def increaseFallback(self, release):
+        """bump trailing .<self.string><DIGIT> or add .<self.string>1"""
+        string = self.string
+        if string is None:
+            string = ""
+        relre = re.compile(r'(?P<prefix>.+\.)' + re.escape(string) +
+                           r'(?P<post>\d+$)')
+        relmatch = relre.search(release)
+        if relmatch:
+            prefix = relmatch.group('prefix')
+            post = relmatch.group('post')
+            new = prefix + string + self.increaseMain(post)
+        else:
+            new = release.rstrip() + '.' + string + '1'
+        return new
+
+    def increase(self, match):
+        old = match.group(1)  # only the release value
+        try:
+            if self.string is not None:
+                new = self.increaseFallback(old)
+            elif old.find('jpp') > 0:
+                new = self.increaseJPP(old)
+            else:
+                new = self.increaseMain(old)
+        except BumpSpecError:
+            new = self.increaseFallback(old)
+        if self.verbose:
+            self.debugdiff(old, new)
+        # group 0 is the full line that defines the release
+        return match.group(0).replace(old, new)
+
+    def writeFile(self, filename):
+        f = open(filename, "w")
+        f.writelines(self.lines)
+        f.close()
+
+    def debugdiff(self, old, new):
+        print ('%s\n-%s\n+%s\n' % (self.filename, old, new))
+
+if __name__ == "__main__":
+    usage = '''Usage: %prog [OPTION]... SPECFILE...
+
+rpmdev-bumpspec bumps release tags in specfiles.'''
+
+    version = '''rpmdev-bumpspec version %s
+
+Copyright (c) 2005-2014 Fedora Project
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.''' % __version__
+
+    userstring = subprocess.Popen("rpmdev-packager 2>/dev/null", shell=True,
+                                  stdout=subprocess.PIPE).communicate()[0]
+    if sys.version_info[0] > 2:
+        userstring = userstring.decode(errors='replace')
+    userstring = userstring.strip() or None
+
+    parser = OptionParser(usage=usage)
+    parser.add_option("-c", "--comment",
+                      help="changelog comment (default: \"- rebuilt\")")
+    parser.add_option("-u", "--userstring", default=userstring,
+                      help="user name+email string (default: output from "
+                      "rpmdev-packager(1))")
+    parser.add_option("-r", "--rightmost", default=False, action='store_true',
+                      help="bump trailing .<DIGIT> component if found, "
+                      "append .1 if not; no-op if -s is specified")
+    parser.add_option("-s", "--string", default=None,
+                      help="bump trailing .STRING<DIGIT> component if found, "
+                      "append .STRING1 if not; trumps -r")
+    parser.add_option("-n", "--new",
+                      help="set new version and reset/set release "
+                      "(simple spec files only)")
+    parser.add_option("-V", "--verbose", default=False, action='store_true',
+                      help="more output")
+    parser.add_option("-v", "--version", default=False, action='store_true',
+                      help="output version number and exit")
+    (opts, args) = parser.parse_args()
+
+    if opts.version:
+        print (version)
+        sys.exit(0)
+
+    if not args:
+        parser.error('No specfiles specified')
+
+    if not opts.userstring:
+        parser.error('Userstring required, see option -u')
+
+    if not opts.comment:
+        opts.comment = '- new version' if opts.new else '- rebuilt'
+
+    # Grab bullet, insert one if not found.
+    bullet_re = re.compile(r'^([^\s\w])\s', re.UNICODE)
+    bullet = "-"
+    match = bullet_re.search(opts.comment)
+    if match:
+        bullet = match.group(1)
+    else:
+        opts.comment = bullet + " " + opts.comment
+
+    # Format comment.
+    if opts.comment.find("\n") == -1:
+        wrapopts = {"subsequent_indent": (len(bullet)+1) * " ",
+                    "break_long_words":  False}
+        if sys.version_info[:2] > (2, 5):
+            wrapopts["break_on_hyphens"] = False
+        opts.comment = textwrap.fill(opts.comment, 80, **wrapopts)
+
+    # Prepare release component string.
+    string = opts.string
+    if string is None and opts.rightmost:
+        string = ""
+
+    for aspec in args:
+        try:
+            s = SpecFile(aspec, opts.verbose, string)
+        except:
+            # Not actually a parser error, but... meh.
+            parser.error(sys.exc_info()[1])
+        if opts.new:
+            s.newVersion(opts.new)
+        else:
+            s.bumpRelease()
+        s.writeFile(aspec)
+
+        # Get EVR for changelog entry.
+        cmd = ("rpm", "-q", "--specfile", "--define", "dist %{nil}",
+               "--qf=%|epoch?{%{epoch}:}:{}|%{version}-%{release}\n", aspec)
+        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        evr = popen.communicate()[0].split(b"\n")[0]
+        if sys.version_info[0] > 2:
+            evr = evr.decode(errors='replace')
+
+        s.addChangelogEntry(evr, opts.comment, opts.userstring)
+        s.writeFile(aspec)
+
+sys.exit(0)

roles/pdc/backend/files/rpmmacros

@@ -0,0 +1,5 @@
+# rpmmacros for the-new-hotness to build source rpms in place in /var/tmp/
+
+%_topdir .
+%_sourcedir .
+%_srcrpmdir .

roles/pdc/backend/files/yumconfig

@@ -0,0 +1,35 @@
+[main]
+cachedir=/var/cache/yum-hotness/$basearch/$releasever
+keepcache=0
+debuglevel=2
+logfile=/var/log/yum-hotness.log
+exactarch=1
+obsoletes=1
+gpgcheck=1
+plugins=1
+installonly_limit=3
+
+# It is usually 90m.  We'll expire often..
+metadata_expire=5m
+
+# We have three repos to query...
+[rawhide-x86_64]
+name=Rawhide x86_64
+failovermethod=priority
+baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/x86_64/
+enabled=1
+gpgcheck=0
+
+[rawhide-i386]
+name=Rawhide i386
+failovermethod=priority
+baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/i386/
+enabled=1
+gpgcheck=0
+
+[rawhide-armhfp]
+name=Rawhide armhfp
+failovermethod=priority
+baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/armhfp/
+enabled=1
+gpgcheck=0

roles/pdc/backend/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+# Configuration for the pdc-updater backend consumer
+
+- name: install needed packages
+  yum: pkg={{ item }} state=present
+  with_items:
+  - pdc-updater
+  tags: pdc
+
+- name: copy database configuration
+  template: >
+    src={{ item }} dest=/etc/fedmsg.d/{{ item }}
+    owner=fedmsg group=fedmsg mode=0600
+  with_items:
+  - pdcupdater.py
+  notify: restart fedmsg-hub
+  tags: pdc

roles/pdc/backend/templates/pdcupdater.py

@@ -0,0 +1,76 @@
+# Configuration for the pdc-updater backend.
+
+config = {
+    # Should we turn on the realtime updater?
+    'pdcupdater.enabled': True,
+
+    # Credentials to talk to PDC
+    'pdcupdater.pdc': {
+        {% if env == 'staging' %}
+        'server': 'https://apps.stg.fedoraproject.org/pdc/rest_api/v1/',
+        'insecure': False,
+        'token': '{{pdc_updater_api_token_stg }}',
+        {% else %}
+        'server': 'https://apps.fedoraproject.org/pdc/rest_api/v1/',
+        'insecure': False,
+        'token': '{{pdc_updater_api_token_prod }}',
+        {% endif %}
+    },
+
+    # Credentials to talk to FAS
+    'pdcupdater.fas': {
+        {% if env == 'staging' %}
+        'base_url': 'https://admin.stg.fedoraproject.org/accounts',
+        {% else %}
+        'base_url': 'https://admin.fedoraproject.org/accounts',
+        {% endif %}
+        'username': '{{ fedoraDummyUser }}',
+        'password': '{{ fedoraDummyUserPassword }}',
+    },
+
+    # PkgDB details
+    {% if env == 'staging' %}
+    'pdcupdater.pkgdb_url': 'https://admin.stg.fedoraproject.org/pkgdb',
+    {% else %}
+    'pdcupdater.pkgdb_url': 'https://admin.fedoraproject.org/pkgdb',
+    {% endif %}
+
+    # Koji details
+    {% if env == 'staging' %}
+    'pdcupdater.koji_url': 'http://koji.stg.fedoraproject.org/kojihub',
+    {% else %}
+    'pdcupdater.koji_url': 'http://koji.fedoraproject.org/kojihub',
+    {% endif %}
+
+    # Where to find composes
+    {% if env == 'staging' %}
+    'pdcupdater.old_composes_url': 'https://kojipkgs.stg.fedoraproject.org/compose/',
+    {% else %}
+    'pdcupdater.old_composes_url': 'https://kojipkgs.fedoraproject.org/compose/',
+    {% endif %}
+
+    # We have an explicit list of these in the config so we can turn them on
+    # and off individually in production if one is causing an issue.
+    'pdcupdater.handlers': [
+        'pdcupdater.handlers.pkgdb:NewPackageHandler',
+        'pdcupdater.handlers.pkgdb:NewPackageBranchHandler',
+        'pdcupdater.handlers.rpms:NewRPMHandler',
+        'pdcupdater.handlers.compose:NewComposeHandler',
+        'pdcupdater.handlers.persons:NewPersonHandler',
+    ],
+
+    'logging': dict(
+        loggers=dict(
+            pdcupdater={
+                "level": "DEBUG",
+                "propagate": False,
+                "handlers": ["console"],
+            },
+            requests={
+                "level": "INFO",
+                "propagate": False,
+                "handlers": ["console"],
+            },
+        )
+    )
+}

roles/pdc/frontend/tasks/main.yml

@@ -25,27 +25,29 @@
   - patternfly1
   - pdc-server
   - xmlsec1
-  tags:
+  tags: pdc
-  - pdc
 
 - name: Copy over settings_local.py
   template: src=settings_local.py dest=/usr/lib/python2.7/site-packages/pdc/settings_local.py
   notify: reload httpd
-  tags:
+  tags: pdc
-  - pdc
 
 - name: Copy over httpd config
   template: src=pdc.conf dest=/etc/httpd/conf.d/pdc.conf
   notify: reload httpd
+  tags: pdc
+
+- name: ensure selinux lets httpd talk to postgres
+  seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
   tags:
   - pdc
+  - selinux
 
 - name: create /etc/httpd/saml2
   file: state=directory
         path=/etc/httpd/saml2
         owner=apache group=apache mode=0775
-  tags:
+  tags: pdc
-  - pdc
 
 - name: Install saml2 xml files
   copy: >
@@ -54,8 +56,7 @@
   with_items:
     - metadata.xml
     - idp-metadata.xml
-  tags:
+  tags: pdc
-  - pdc
 
 - name: Install saml2 certs
   copy: >
@@ -64,11 +65,4 @@
   with_items:
     - pdc.fedorainfracloud.org.pem
     - pdc.fedorainfracloud.org.key
-  tags:
+  tags: pdc
-  - pdc
-
-- name: ensure selinux lets httpd talk to postgres
-  seboolean: name=httpd_can_network_connect_db persistent=yes state=yes
-  tags:
-  - pdc
-  - selinux