From a93ec459da8af690279fe0bc4a8cb846b6a021bf Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Jan 2016 10:40:22 -0500 Subject: [PATCH] A start at pdc config. still need to work out saml2/mellon stuff. --- playbooks/groups/pdc-web.yml | 5 + playbooks/groups/pdc.yml | 65 ++++ playbooks/hosts/pdc.fedorainfracloud.org.yml | 2 +- .../pdc/backend/files/hotfix/rpmdev-bumpspec | 336 ++++++++++++++++++ roles/pdc/backend/files/rpmmacros | 5 + roles/pdc/backend/files/yumconfig | 35 ++ roles/pdc/backend/tasks/main.yml | 17 + roles/pdc/backend/templates/pdcupdater.py | 76 ++++ .../pdc/{ => frontend}/files/idp-metadata.xml | 0 roles/pdc/{ => frontend}/files/metadata.xml | 0 .../files/patternfly-patternfly1-epel-7.repo | 0 .../{ => frontend}/files/xchu-pdc-epel-7.repo | 0 roles/pdc/{ => frontend}/tasks/main.yml | 30 +- roles/pdc/{ => frontend}/templates/pdc.conf | 0 .../templates/settings_local.py | 0 15 files changed, 552 insertions(+), 19 deletions(-) create mode 100644 playbooks/groups/pdc-web.yml create mode 100644 playbooks/groups/pdc.yml create mode 100755 roles/pdc/backend/files/hotfix/rpmdev-bumpspec create mode 100644 roles/pdc/backend/files/rpmmacros create mode 100644 roles/pdc/backend/files/yumconfig create mode 100644 roles/pdc/backend/tasks/main.yml create mode 100644 roles/pdc/backend/templates/pdcupdater.py rename roles/pdc/{ => frontend}/files/idp-metadata.xml (100%) rename roles/pdc/{ => frontend}/files/metadata.xml (100%) rename roles/pdc/{ => frontend}/files/patternfly-patternfly1-epel-7.repo (100%) rename roles/pdc/{ => frontend}/files/xchu-pdc-epel-7.repo (100%) rename roles/pdc/{ => frontend}/tasks/main.yml (88%) rename roles/pdc/{ => frontend}/templates/pdc.conf (100%) rename roles/pdc/{ => frontend}/templates/settings_local.py (100%) diff --git a/playbooks/groups/pdc-web.yml b/playbooks/groups/pdc-web.yml new file mode 100644 index 0000000000..79d0d078ba --- /dev/null +++ b/playbooks/groups/pdc-web.yml @@ -0,0 +1,5 @@ +# create a new notifs-web server +# NOTE: should be used with --limit most of the time +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/notifs-web* or from hostvars + diff --git a/playbooks/groups/pdc.yml b/playbooks/groups/pdc.yml new file mode 100644 index 0000000000..98f28366fe --- /dev/null +++ b/playbooks/groups/pdc.yml @@ -0,0 +1,65 @@ +# PDC servers (both frontend and backend) + +- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=pdc-web:pdc-web-stg:pdc-backend:pdc-backend-stg" + +- name: dole out the generic configuration + user: root + gather_facts: True + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + handlers: + - include: "{{ handlers }}/restart_services.yml" + + roles: + - base + - rkhunter + - nagios_client + - collectd/base + - hosts + - fas_client + - sudo + - fedmsg/base + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + +- name: stuff for the web nodes + hosts: notifs-web;notifs-web-stg + user: root + gather_facts: True + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + handlers: + - include: "{{ handlers }}/restart_services.yml" + + roles: + - role: openvpn/client + when: env != "staging" + - apache + - pdc/frontend + + tasks: + - include: "{{ tasks }}/mod_wsgi.yml" + +- name: stuff just for the backend nodes + hosts: pdc-backend;pdc-backend-stg + user: root + gather_facts: True + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + handlers: + - include: "{{ handlers }}/restart_services.yml" + + roles: + - fedmsg/hub + - pdc/backend + - role: collectd/fedmsg-service + process: fedmsg-hub diff --git a/playbooks/hosts/pdc.fedorainfracloud.org.yml b/playbooks/hosts/pdc.fedorainfracloud.org.yml index 2935f8ceb1..3c26ebceee 100644 --- a/playbooks/hosts/pdc.fedorainfracloud.org.yml +++ b/playbooks/hosts/pdc.fedorainfracloud.org.yml @@ -31,7 +31,7 @@ roles: - base - postgresql_server - - pdc + - pdc/frontend handlers: - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/pdc/backend/files/hotfix/rpmdev-bumpspec b/roles/pdc/backend/files/hotfix/rpmdev-bumpspec new file mode 100755 index 0000000000..916eb31a91 --- /dev/null +++ b/roles/pdc/backend/files/hotfix/rpmdev-bumpspec @@ -0,0 +1,336 @@ +#!/usr/bin/python -tt +# -*- mode: Python; indent-tabs-mode: nil; coding: utf-8 -*- +# +# Copyright (c) 2005-2014 Fedora Project +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +import re +import subprocess +import sys +import textwrap +import time +from optparse import OptionParser + + +__version__ = "1.0.13" + + +class BumpSpecError(Exception): + pass + + +class SpecFile: + def __init__(self, filename, verbose=False, string=None): + self.verbose = verbose + self.string = string + + self.filename = filename + f = None + try: + f = open(filename, "r") + self.lines = f.readlines() + finally: + f and f.close() + + # supported release value macro definitions + _macro_bump_patterns = ( + re.compile(r"^%(?:define|global)\s+(?i)release\s+(\d+.*)"), + re.compile(r"^%(?:define|global)\s+(?i)baserelease\s+(\d+.*)"), + ) + # normal "Release:" tag lines + _tag_bump_patterns = ( + re.compile(r"^Release\s*:\s*(\d+.*)", re.I), + re.compile(r"^Release\s*:\s+%release_func\s+(\d+.*)", re.I), + ) + # lines we don't want to mess with + _skip_patterns = ( + re.compile(r"\$Revision:"), + ) + + def bumpRelease(self): + # remember whether we've bumped a macro definition + bumped_macro = False + # count how many times/lines we've bumped + bumped = 0 + + for i in range(len(self.lines)): + # If we've bumped a macro, we assume this is enough for + # the rest of the spec file, so we don't bump a macro and + # a corresponding Release tag. The macro may or may not be + # used for the definition of one or more Release tags. + # Macro-madness makes that hard to check for. + if bumped_macro: + break + + skipped = False + for pattern in SpecFile._skip_patterns: + if pattern.search(self.lines[i]): + skipped = True + break + if skipped: + continue + + for pattern in SpecFile._macro_bump_patterns: + (self.lines[i], n) = \ + pattern.subn(self.increase, self.lines[i], 1) + if n: # this pattern has lead to a change + bumped += 1 + bumped_macro = True + break + else: # no pattern matched + for pattern in SpecFile._tag_bump_patterns: + (self.lines[i], n) = \ + pattern.subn(self.increase, self.lines[i], 1) + if n: # this pattern has lead to a change + bumped += 1 + break + else: # no pattern matched at all + # Bump ^Release: ... line least-significant. + if self.lines[i].lower().startswith('release:'): + old = self.lines[i][len('Release:'):].rstrip() + new = self.increaseFallback(old) + if self.verbose: + self.debugdiff(old, new) + if old != new: + self.lines[i] = self.lines[i].replace(old, new) + bumped += 1 + + if bumped: + return + if self.verbose: + sys.stderr.write('ERROR: No release value matched: %s\n' % + self.filename) + sys.exit(1) + + def newVersion(self, vr): + rpos = vr.find('-') + if rpos >= 0: # set custom Release value + r = vr[rpos+1:] + v = vr[:rpos] + else: + r = "1%{?dist}" + v = vr + for i in range(len(self.lines)): + if self.lines[i].lower().startswith('version:'): + self.lines[i] = re.sub( + r'[^: \t]*$', v, self.lines[i].rstrip()) + '\n' + elif self.lines[i].lower().startswith('release:'): + self.lines[i] = re.sub( + r'[^: \t]*$', r, self.lines[i].rstrip()) + '\n' + + _changelog_pattern = re.compile(r"^%changelog(\s|$)", re.I) + + def addChangelogEntry(self, evr, entry, email): + for i in range(len(self.lines)): + if SpecFile._changelog_pattern.match(self.lines[i]): + if len(evr): + evrstring = ' - %s' % evr + else: + evrstring = '' + date = time.strftime("%a %b %d %Y", time.gmtime()) + newchangelogentry = "* %s %s%s\n%s\n\n" % \ + (date, email, evrstring, entry) + self.lines[i] += newchangelogentry + return + + _main_pre_pattern = re.compile(r'^0\.(?P\d+)(?P.*)') + _main_pattern = re.compile(r'^(?P\d+)(?P.*)') + + def increaseMain(self, release): + if release.startswith('0.'): + relre = SpecFile._main_pre_pattern + pre = True + else: + relre = SpecFile._main_pattern + pre = False + relmatch = relre.search(release) + if not relmatch: # pattern match failed + raise BumpSpecError + value = str(int(relmatch.group('rel')) + 1) + post = relmatch.group('post') + + new = value + post + if not pre: + if post.find('rc') >= 0: + sys.stderr.write( + 'WARNING: Bad pre-release versioning scheme: %s\n' % + self.filename) + raise BumpSpecError + else: + new = '0.' + new + return new + + _jpp_pattern = \ + re.compile(r'(?P.*)(?P\d+)(?Pjpp\.)(?P.*)') + + def increaseJPP(self, release): + """Fedora jpackage release versioning scheme""" + + relmatch = SpecFile._jpp_pattern.search(release) + if not relmatch: # pattern match failed + sys.stderr.write( + 'WARNING: Bad Fedora jpackage release versioning scheme: %s\n' + % self.filename) + raise BumpSpecError + + prefix = relmatch.group('prefix') + value = int(relmatch.group('rel')) + jpp = relmatch.group('jpp') + post = relmatch.group('post') + + newpost = self.increaseMain(post) + new = prefix+str(value)+jpp+newpost + return new + + def increaseFallback(self, release): + """bump trailing . or add .1""" + string = self.string + if string is None: + string = "" + relre = re.compile(r'(?P.+\.)' + re.escape(string) + + r'(?P\d+$)') + relmatch = relre.search(release) + if relmatch: + prefix = relmatch.group('prefix') + post = relmatch.group('post') + new = prefix + string + self.increaseMain(post) + else: + new = release.rstrip() + '.' + string + '1' + return new + + def increase(self, match): + old = match.group(1) # only the release value + try: + if self.string is not None: + new = self.increaseFallback(old) + elif old.find('jpp') > 0: + new = self.increaseJPP(old) + else: + new = self.increaseMain(old) + except BumpSpecError: + new = self.increaseFallback(old) + if self.verbose: + self.debugdiff(old, new) + # group 0 is the full line that defines the release + return match.group(0).replace(old, new) + + def writeFile(self, filename): + f = open(filename, "w") + f.writelines(self.lines) + f.close() + + def debugdiff(self, old, new): + print ('%s\n-%s\n+%s\n' % (self.filename, old, new)) + +if __name__ == "__main__": + usage = '''Usage: %prog [OPTION]... SPECFILE... + +rpmdev-bumpspec bumps release tags in specfiles.''' + + version = '''rpmdev-bumpspec version %s + +Copyright (c) 2005-2014 Fedora Project +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version.''' % __version__ + + userstring = subprocess.Popen("rpmdev-packager 2>/dev/null", shell=True, + stdout=subprocess.PIPE).communicate()[0] + if sys.version_info[0] > 2: + userstring = userstring.decode(errors='replace') + userstring = userstring.strip() or None + + parser = OptionParser(usage=usage) + parser.add_option("-c", "--comment", + help="changelog comment (default: \"- rebuilt\")") + parser.add_option("-u", "--userstring", default=userstring, + help="user name+email string (default: output from " + "rpmdev-packager(1))") + parser.add_option("-r", "--rightmost", default=False, action='store_true', + help="bump trailing . component if found, " + "append .1 if not; no-op if -s is specified") + parser.add_option("-s", "--string", default=None, + help="bump trailing .STRING component if found, " + "append .STRING1 if not; trumps -r") + parser.add_option("-n", "--new", + help="set new version and reset/set release " + "(simple spec files only)") + parser.add_option("-V", "--verbose", default=False, action='store_true', + help="more output") + parser.add_option("-v", "--version", default=False, action='store_true', + help="output version number and exit") + (opts, args) = parser.parse_args() + + if opts.version: + print (version) + sys.exit(0) + + if not args: + parser.error('No specfiles specified') + + if not opts.userstring: + parser.error('Userstring required, see option -u') + + if not opts.comment: + opts.comment = '- new version' if opts.new else '- rebuilt' + + # Grab bullet, insert one if not found. + bullet_re = re.compile(r'^([^\s\w])\s', re.UNICODE) + bullet = "-" + match = bullet_re.search(opts.comment) + if match: + bullet = match.group(1) + else: + opts.comment = bullet + " " + opts.comment + + # Format comment. + if opts.comment.find("\n") == -1: + wrapopts = {"subsequent_indent": (len(bullet)+1) * " ", + "break_long_words": False} + if sys.version_info[:2] > (2, 5): + wrapopts["break_on_hyphens"] = False + opts.comment = textwrap.fill(opts.comment, 80, **wrapopts) + + # Prepare release component string. + string = opts.string + if string is None and opts.rightmost: + string = "" + + for aspec in args: + try: + s = SpecFile(aspec, opts.verbose, string) + except: + # Not actually a parser error, but... meh. + parser.error(sys.exc_info()[1]) + if opts.new: + s.newVersion(opts.new) + else: + s.bumpRelease() + s.writeFile(aspec) + + # Get EVR for changelog entry. + cmd = ("rpm", "-q", "--specfile", "--define", "dist %{nil}", + "--qf=%|epoch?{%{epoch}:}:{}|%{version}-%{release}\n", aspec) + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE) + evr = popen.communicate()[0].split(b"\n")[0] + if sys.version_info[0] > 2: + evr = evr.decode(errors='replace') + + s.addChangelogEntry(evr, opts.comment, opts.userstring) + s.writeFile(aspec) + +sys.exit(0) diff --git a/roles/pdc/backend/files/rpmmacros b/roles/pdc/backend/files/rpmmacros new file mode 100644 index 0000000000..218a52a64c --- /dev/null +++ b/roles/pdc/backend/files/rpmmacros @@ -0,0 +1,5 @@ +# rpmmacros for the-new-hotness to build source rpms in place in /var/tmp/ + +%_topdir . +%_sourcedir . +%_srcrpmdir . diff --git a/roles/pdc/backend/files/yumconfig b/roles/pdc/backend/files/yumconfig new file mode 100644 index 0000000000..205622c7d5 --- /dev/null +++ b/roles/pdc/backend/files/yumconfig @@ -0,0 +1,35 @@ +[main] +cachedir=/var/cache/yum-hotness/$basearch/$releasever +keepcache=0 +debuglevel=2 +logfile=/var/log/yum-hotness.log +exactarch=1 +obsoletes=1 +gpgcheck=1 +plugins=1 +installonly_limit=3 + +# It is usually 90m. We'll expire often.. +metadata_expire=5m + +# We have three repos to query... +[rawhide-x86_64] +name=Rawhide x86_64 +failovermethod=priority +baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/x86_64/ +enabled=1 +gpgcheck=0 + +[rawhide-i386] +name=Rawhide i386 +failovermethod=priority +baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/i386/ +enabled=1 +gpgcheck=0 + +[rawhide-armhfp] +name=Rawhide armhfp +failovermethod=priority +baseurl=https://kojipkgs.fedoraproject.org/repos/rawhide/latest/armhfp/ +enabled=1 +gpgcheck=0 diff --git a/roles/pdc/backend/tasks/main.yml b/roles/pdc/backend/tasks/main.yml new file mode 100644 index 0000000000..0a5084a951 --- /dev/null +++ b/roles/pdc/backend/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# Configuration for the pdc-updater backend consumer + +- name: install needed packages + yum: pkg={{ item }} state=present + with_items: + - pdc-updater + tags: pdc + +- name: copy database configuration + template: > + src={{ item }} dest=/etc/fedmsg.d/{{ item }} + owner=fedmsg group=fedmsg mode=0600 + with_items: + - pdcupdater.py + notify: restart fedmsg-hub + tags: pdc diff --git a/roles/pdc/backend/templates/pdcupdater.py b/roles/pdc/backend/templates/pdcupdater.py new file mode 100644 index 0000000000..e53a2149c2 --- /dev/null +++ b/roles/pdc/backend/templates/pdcupdater.py @@ -0,0 +1,76 @@ +# Configuration for the pdc-updater backend. + +config = { + # Should we turn on the realtime updater? + 'pdcupdater.enabled': True, + + # Credentials to talk to PDC + 'pdcupdater.pdc': { + {% if env == 'staging' %} + 'server': 'https://apps.stg.fedoraproject.org/pdc/rest_api/v1/', + 'insecure': False, + 'token': '{{pdc_updater_api_token_stg }}', + {% else %} + 'server': 'https://apps.fedoraproject.org/pdc/rest_api/v1/', + 'insecure': False, + 'token': '{{pdc_updater_api_token_prod }}', + {% endif %} + }, + + # Credentials to talk to FAS + 'pdcupdater.fas': { + {% if env == 'staging' %} + 'base_url': 'https://admin.stg.fedoraproject.org/accounts', + {% else %} + 'base_url': 'https://admin.fedoraproject.org/accounts', + {% endif %} + 'username': '{{ fedoraDummyUser }}', + 'password': '{{ fedoraDummyUserPassword }}', + }, + + # PkgDB details + {% if env == 'staging' %} + 'pdcupdater.pkgdb_url': 'https://admin.stg.fedoraproject.org/pkgdb', + {% else %} + 'pdcupdater.pkgdb_url': 'https://admin.fedoraproject.org/pkgdb', + {% endif %} + + # Koji details + {% if env == 'staging' %} + 'pdcupdater.koji_url': 'http://koji.stg.fedoraproject.org/kojihub', + {% else %} + 'pdcupdater.koji_url': 'http://koji.fedoraproject.org/kojihub', + {% endif %} + + # Where to find composes + {% if env == 'staging' %} + 'pdcupdater.old_composes_url': 'https://kojipkgs.stg.fedoraproject.org/compose/', + {% else %} + 'pdcupdater.old_composes_url': 'https://kojipkgs.fedoraproject.org/compose/', + {% endif %} + + # We have an explicit list of these in the config so we can turn them on + # and off individually in production if one is causing an issue. + 'pdcupdater.handlers': [ + 'pdcupdater.handlers.pkgdb:NewPackageHandler', + 'pdcupdater.handlers.pkgdb:NewPackageBranchHandler', + 'pdcupdater.handlers.rpms:NewRPMHandler', + 'pdcupdater.handlers.compose:NewComposeHandler', + 'pdcupdater.handlers.persons:NewPersonHandler', + ], + + 'logging': dict( + loggers=dict( + pdcupdater={ + "level": "DEBUG", + "propagate": False, + "handlers": ["console"], + }, + requests={ + "level": "INFO", + "propagate": False, + "handlers": ["console"], + }, + ) + ) +} diff --git a/roles/pdc/files/idp-metadata.xml b/roles/pdc/frontend/files/idp-metadata.xml similarity index 100% rename from roles/pdc/files/idp-metadata.xml rename to roles/pdc/frontend/files/idp-metadata.xml diff --git a/roles/pdc/files/metadata.xml b/roles/pdc/frontend/files/metadata.xml similarity index 100% rename from roles/pdc/files/metadata.xml rename to roles/pdc/frontend/files/metadata.xml diff --git a/roles/pdc/files/patternfly-patternfly1-epel-7.repo b/roles/pdc/frontend/files/patternfly-patternfly1-epel-7.repo similarity index 100% rename from roles/pdc/files/patternfly-patternfly1-epel-7.repo rename to roles/pdc/frontend/files/patternfly-patternfly1-epel-7.repo diff --git a/roles/pdc/files/xchu-pdc-epel-7.repo b/roles/pdc/frontend/files/xchu-pdc-epel-7.repo similarity index 100% rename from roles/pdc/files/xchu-pdc-epel-7.repo rename to roles/pdc/frontend/files/xchu-pdc-epel-7.repo diff --git a/roles/pdc/tasks/main.yml b/roles/pdc/frontend/tasks/main.yml similarity index 88% rename from roles/pdc/tasks/main.yml rename to roles/pdc/frontend/tasks/main.yml index 2bd33c0ad2..7564e32127 100644 --- a/roles/pdc/tasks/main.yml +++ b/roles/pdc/frontend/tasks/main.yml @@ -25,27 +25,29 @@ - patternfly1 - pdc-server - xmlsec1 - tags: - - pdc + tags: pdc - name: Copy over settings_local.py template: src=settings_local.py dest=/usr/lib/python2.7/site-packages/pdc/settings_local.py notify: reload httpd - tags: - - pdc + tags: pdc - name: Copy over httpd config template: src=pdc.conf dest=/etc/httpd/conf.d/pdc.conf notify: reload httpd + tags: pdc + +- name: ensure selinux lets httpd talk to postgres + seboolean: name=httpd_can_network_connect_db persistent=yes state=yes tags: - pdc + - selinux - name: create /etc/httpd/saml2 file: state=directory - path=/etc/httpd/saml2 - owner=apache group=apache mode=0775 - tags: - - pdc + path=/etc/httpd/saml2 + owner=apache group=apache mode=0775 + tags: pdc - name: Install saml2 xml files copy: > @@ -54,8 +56,7 @@ with_items: - metadata.xml - idp-metadata.xml - tags: - - pdc + tags: pdc - name: Install saml2 certs copy: > @@ -64,11 +65,4 @@ with_items: - pdc.fedorainfracloud.org.pem - pdc.fedorainfracloud.org.key - tags: - - pdc - -- name: ensure selinux lets httpd talk to postgres - seboolean: name=httpd_can_network_connect_db persistent=yes state=yes - tags: - - pdc - - selinux + tags: pdc diff --git a/roles/pdc/templates/pdc.conf b/roles/pdc/frontend/templates/pdc.conf similarity index 100% rename from roles/pdc/templates/pdc.conf rename to roles/pdc/frontend/templates/pdc.conf diff --git a/roles/pdc/templates/settings_local.py b/roles/pdc/frontend/templates/settings_local.py similarity index 100% rename from roles/pdc/templates/settings_local.py rename to roles/pdc/frontend/templates/settings_local.py