From f6d0b0dee017e99834ea07adb642c154af533dd7 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 21 Sep 2015 19:57:01 +0000 Subject: [PATCH] Add a bunch more things to batcave01. --- playbooks/groups/batcave.yml | 1 + roles/batcave/files/allows | 97 +++ .../batcave/files/fedmsg-announce-commits.py | 141 ++++ roles/batcave/files/git-notifier | 553 +++++++++++++ .../infrastructure.fedoraproject.org.conf | 206 +++++ roles/batcave/files/mime-types.conf | 1 + roles/batcave/files/oshift_mod.py | 742 ++++++++++++++++++ .../files/retrieve-security-question.py | 108 +++ roles/batcave/files/sync-openshift-keys.py | 137 ++++ .../batcave/files/zodbot-announce-commits.py | 138 ++++ roles/batcave/tasks/main.yml | 110 ++- .../templates/sync-openshift-keys.conf.j2 | 13 + 12 files changed, 2232 insertions(+), 15 deletions(-) create mode 100644 roles/batcave/files/allows create mode 100644 roles/batcave/files/fedmsg-announce-commits.py create mode 100755 roles/batcave/files/git-notifier create mode 100644 roles/batcave/files/infrastructure.fedoraproject.org.conf create mode 100644 roles/batcave/files/mime-types.conf create mode 100644 roles/batcave/files/oshift_mod.py create mode 100755 roles/batcave/files/retrieve-security-question.py create mode 100755 roles/batcave/files/sync-openshift-keys.py create mode 100644 roles/batcave/files/zodbot-announce-commits.py create mode 100644 roles/batcave/templates/sync-openshift-keys.conf.j2 diff --git a/playbooks/groups/batcave.yml b/playbooks/groups/batcave.yml index f291838547..30920aa1d2 100644 --- a/playbooks/groups/batcave.yml +++ b/playbooks/groups/batcave.yml @@ -24,6 +24,7 @@ - cgit/base - cgit/clean_lock_cron - cgit/make_pkgs_list + - rsyncd - apache - httpd/mod_ssl - { role: httpd/certificate, name: wildcard-2014.fedoraproject.org, SSLCertificateChainFile: wildcard-2014.fedoraproject.org.intermediate.cert } diff --git a/roles/batcave/files/allows b/roles/batcave/files/allows new file mode 100644 index 0000000000..47b30e7242 --- /dev/null +++ b/roles/batcave/files/allows @@ -0,0 +1,97 @@ +allow from .fedoraproject.org +allow from 10. +allow from 64.34.163.94 +allow from 64.34.163.95 +allow from 64.34.163.96 +allow from 64.34.183.130 +allow from 64.34.184.178 +allow from 64.34.184.179 +allow from 64.34.195.12 +allow from 64.34.195.13 +allow from 64.34.195.14 +allow from 64.34.203.162 +allow from 64.34.212.36 +allow from 64.34.212.37 +allow from 64.34.212.38 +allow from 66.135.52.16 +allow from 66.135.52.17 +allow from 66.135.52.84 +allow from 66.135.52.85 +allow from 66.135.55.241 +allow from 66.135.60.113 +allow from 66.35.62.160/28 +allow from 80.239.144.84 +allow from 80.239.156.208/28 +allow from 94.76.206.175 +allow from 128.197.185.42 +allow from 128.197.185.45 +allow from 140.211.169.192/26 +allow from 209.132.178.0/23 +allow from 209.132.180.0/24 +allow from 209.132.181.0/24 +allow from 209.132.182.51 +allow from 209.132.184.0/24 +allow from 213.175.193.204 +allow from 213.175.193.205 +allow from 213.175.193.206 +allow from 213.175.193.207 + +# ibiblio +allow from 152.19.134.136 +allow from 152.19.134.137 +allow from 152.19.134.138 +allow from 152.19.134.139 +allow from 152.19.134.140 +allow from 152.19.134.141 +allow from 152.19.134.142 +allow from 152.19.134.143 +allow from 152.19.134.144 +allow from 152.19.134.145 +allow from 152.19.134.146 +allow from 152.19.134.147 +allow from 152.19.134.148 +allow from 152.19.134.149 +allow from 152.19.134.150 +allow from 152.19.134.169 +allow from 152.19.134.170 +allow from 152.19.134.172 +allow from 152.19.134.173 +allow from 152.19.134.191 +allow from 152.19.134.192 +allow from 152.19.134.193 +allow from 152.19.134.194 +allow from 152.19.134.195 +allow from 152.19.134.196 +allow from 152.19.134.197 +allow from 152.19.134.198 + +# internetx +allow from 85.236.55.0/28 +allow from 2001:4178:2:1269::/64 + +# ibiblio ipv6 +allow from 2610:28:3090:3001:dead:beef:cafe:fe00/120 +# Private network cloud nat +allow from 209.132.180.6 + +# rdu - rht +allow from 204.85.14.1 +allow from 204.85.14.2 +allow from 204.85.14.3 + +# sourceware systems +allow from 209.132.180.128/27 + +# colocation america system. +allow from 67.203.2.64/29 +allow from 2067:f188:0:0::0/64 + +# host1plus +allow from 5.175.150.48/28 + +# dedicatedsolutions +allow from 67.219.144.66 +allow from 67.219.144.67 +allow from 67.219.144.68 +allow from 67.219.144.69 +allow from 67.219.144.70 diff --git a/roles/batcave/files/fedmsg-announce-commits.py b/roles/batcave/files/fedmsg-announce-commits.py new file mode 100644 index 0000000000..cbec69b261 --- /dev/null +++ b/roles/batcave/files/fedmsg-announce-commits.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +import getpass +import os +import subprocess as sp +import sys + +from collections import defaultdict + +import pygit2 +import six + +import fedmsg +import fedmsg.config + +# Use $GIT_DIR to determine where this repo is. +abspath = os.path.abspath(os.environ['GIT_DIR']) +# This assumes git root dir is named "repo_name.git" +repo_name = '.'.join(abspath.split(os.path.sep)[-1].split('.')[:-1]) + +username = getpass.getuser() + +repo = pygit2.Repository(abspath) + +print("Emitting a message to the fedmsg bus.") +config = fedmsg.config.load_config([], None) +config['active'] = True +config['endpoints']['relay_inbound'] = config['relay_inbound'] +fedmsg.init(name='relay_inbound', cert_prefix='scm', **config) + + +def revs_between(head, base): + """ Yield revisions between HEAD and BASE. """ + + # pygit2 can't do a rev-list yet, so we have to shell out.. silly. + cmd = '/usr/bin/git rev-list %s...%s' % (head.id, base.id) + proc = sp.Popen(cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE, cwd=abspath) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + raise IOError('git rev-list failed: %r, err: %r' % (stdout, stderr)) + + for line in stdout.strip().split('\n'): + yield line.strip() + + +def build_stats(commit): + files = defaultdict(lambda: defaultdict(int)) + + # Calculate diffs against all parent commits + diffs = [repo.diff(parent, commit) for parent in commit.parents] + # Unless this is the first commit, with no parents. + diffs = diffs or [commit.tree.diff_to_tree(swap=True)] + + for diff in diffs: + for patch in diff: + path = patch.new_file_path + files[path]['additions'] += patch.additions + files[path]['deletions'] += patch.deletions + files[path]['lines'] += patch.additions + patch.deletions + + total = defaultdict(int) + for name, stats in files.items(): + total['additions'] += stats['additions'] + total['deletions'] += stats['deletions'] + total['lines'] += stats['lines'] + total['files'] += 1 + + return files, total + + +seen = [] + +# Read in all the rev information git-receive-pack hands us. +lines = [line.split() for line in sys.stdin.readlines()] +for line in lines: + base, head, branch = line + branch = '/'.join(branch.split('/')[2:]) + + try: + head = repo.revparse_single(head) + except KeyError: + # This means they are deleting this branch.. and we don't have a fedmsg + # for that (yet?). It is disallowed by dist-git in Fedora anyways. + continue + + try: + base = repo.revparse_single(base) + revs = revs_between(head, base) + except KeyError: + revs = [head.id] + + def _build_commit(rev): + commit = repo.revparse_single(six.text_type(rev)) + + # Tags are a little funny, and vary between versions of pygit2, so we'll + # just ignore them as far as fedmsg is concerned. + if isinstance(commit, pygit2.Tag): + return None + + files, total = build_stats(commit) + + return dict( + name=commit.author.name, + email=commit.author.email, + username=username, + summary=commit.message.split('\n')[0], + message=commit.message, + stats=dict( + files=files, + total=total, + ), + rev=six.text_type(rev), + path=abspath, + repo=repo_name, + branch=branch, + agent=os.getlogin(), + ) + + commits = map(_build_commit, revs) + + print("* Publishing information for %i commits" % len(commits)) + for commit in reversed(commits): + if commit is None: + continue + + # Keep track of whether or not we have already published this commit on + # another branch or not. It is conceivable that someone could make a + # commit to a number of branches, and push them all at the same time. + # Make a note in the fedmsg payload so we can try to reduce spam at a + # later stage. + if commit['rev'] in seen: + commit['seen'] = True + else: + commit['seen'] = False + seen.append(commit['rev']) + + fedmsg.publish( + topic="receive", + msg=dict(commit=commit), + modname="infragit", + ) diff --git a/roles/batcave/files/git-notifier b/roles/batcave/files/git-notifier new file mode 100755 index 0000000000..bb68f3fe81 --- /dev/null +++ b/roles/batcave/files/git-notifier @@ -0,0 +1,553 @@ +#! /usr/bin/env python + +import optparse +import os +import shutil +import socket +import sys +import subprocess +import tempfile +import time + +Name = "git-notifier" +Version = "0.3" +CacheFile = ".%s.dat" % Name +Separator = "\n>---------------------------------------------------------------\n" +NoDiff = "[nodiff]" +NoMail = "[nomail]" + +gitolite = "GL_USER" in os.environ +whoami = os.environ["LOGNAME"] +sender = os.environ["GL_USER"] if gitolite else whoami + +Options = [ + # Name, argument, default, help, + ("allchanges", True, set(), "branches for which *all* changes are to be reported"), + ("debug", False, False, "enable debug output"), + ("diff", True, None, "mail out diffs between two revisions"), + ("emailprefix", True, "[git]", "Subject prefix for mails"), + ("hostname", True, socket.gethostname(), "host where the repository is hosted"), + ("log", True, "%s.log" % Name, "set log output"), + ("mailinglist", True, whoami, "destination address for mails"), + ("manual", True, None, "notifiy for a manually given set of revisions"), + ("maxdiffsize", True, 50, "limit the size of diffs in mails (KB)"), + ("noupdate", False, False, "do not update the state file"), + ("repouri", True, None, "full URI for the repository"), + ("sender", True, sender, "sender address for mails"), + ("link", True, None, "Link to insert into mail, %s will be replaced with revision"), + ("updateonly", False, False, "update state file only, no mails"), + ("users", True, None, "location of a user-to-email mapping file"), + ] + +class State: + def __init__(self): + self.clear() + + def clear(self): + self.heads = {} + self.tags = {} + self.revs = set() + + self.reported = set() # Revs reported this run so far. + + def writeTo(self, file): + if os.path.exists(CacheFile): + try: + shutil.move(CacheFile, CacheFile + ".bak") + except IOError: + pass + + out = open(file, "w") + + for (head, ref) in self.heads.items(): + print >>out, "head", head, ref + + for (tag, ref) in self.tags.items(): + print >>out, "tag", tag, ref + + for rev in self.revs: + print >>out, "rev", rev + + def readFrom(self, file): + self.clear() + + for line in open(file): + + line = line.strip() + if not line or line.startswith("#"): + continue + + m = line.split() + + if len(m) == 3: + (type, key, val) = (m[0], m[1], m[2]) + else: + # No heads. + (type, key, val) = (m[0], m[1], "") + + if type == "head": + self.heads[key] = val + + elif type == "tag": + self.tags[key] = val + + elif type == "rev": + self.revs.add(key) + + else: + error("unknown type %s in cache file" % type) + +class GitConfig: + def __init__(self, args): + self.parseArgs(args) + self.maxdiffsize *= 1024 # KBytes to bytes. + + if self.allchanges and not isinstance(self.allchanges, set): + self.allchanges = set([head.strip() for head in self.allchanges.split(",")]) + + if not self.debug: + self.log = open(self.log, "a") + else: + self.log = sys.stderr + + if not self.users and "GL_ADMINDIR" in os.environ: + users = os.path.join(os.environ["GL_ADMINDIR"], "conf/sender.cfg") + if os.path.exists(users): + self.users = users + + self.readUsers() + + def parseArgs(self, args): + + parser = optparse.OptionParser(version=Version) + + for (name, arg, default, help) in Options: + defval = self._git_config(name, default) + + if isinstance(default, int): + defval = int(defval) + + if not arg: + defval = bool(defval) + + if not arg: + action = "store_true" if not default else "store_false" + parser.add_option("--%s" % name, action=action, dest=name, default=defval, help=help) + + else: + type = "string" if not isinstance(default, int) else "int" + parser.add_option("--%s" % name, action="store", type=type, default=defval, dest=name, help=help) + + (options, args) = parser.parse_args(args) + + if len(args) != 0: + parser.error("incorrect number of arguments") + + for (name, arg, default, help) in Options: + self.__dict__[name] = options.__dict__[name] + + def readUsers(self): + if self.users and os.path.exists(self.users): + for line in open(self.users): + line = line.strip() + if not line or line.startswith("#"): + continue + + m = line.split() + + if self.sender == m[0]: + self.sender = " ".join(m[1:]) + break + + def _git_config(self, key, default): + cfg = git(["config hooks.%s" % key]) + return cfg[0] if cfg else default + +def log(msg): + print >>Config.log, "%s - %s" % (time.asctime(), msg) + +def error(msg): + log("Error: %s" % msg) + sys.exit(1) + +def git(args, stdout_to=subprocess.PIPE, all=False): + if isinstance(args, tuple) or isinstance(args, list): + args = " ".join(args) + + try: + if Config.debug: + print >>sys.stderr, "> git " + args + except NameError: + # Config may not be defined yet. + pass + + try: + child = subprocess.Popen("git " + args, shell=True, stdin=None, stdout=stdout_to, stderr=subprocess.PIPE) + (stdout, stderr) = child.communicate() + except OSError, e: + error("cannot start git: %s" % str(e)) + + if child.returncode != 0 and stderr: + msg = ": %s" % stderr if stderr else "" + error("git child failed with exit code %d%s" % (child.returncode, msg)) + + if stdout_to != subprocess.PIPE: + return [] + + if not all: + return [line.strip() for line in stdout.split("\n") if line] + else: + return stdout.split("\n") + +def getHeads(state): + for (rev, head) in [head.split() for head in git("show-ref --heads")]: + if head.startswith("refs/heads/"): + head = head[11:] + + state.heads[head] = rev + +def getTags(state): + for (rev, tag) in [head.split() for head in git("show-ref --tags")]: + # We are only interested in annotaged tags. + type = git("cat-file -t %s" % rev)[0] + + if type == "tag": + if tag.startswith("refs/tags/"): + tag= tag[10:] + + state.tags[tag] = rev + +def getReachableRefs(state): + for rev in git(["rev-list"] + state.heads.keys() + state.tags.keys()): + state.revs.add(rev) + +def getCurrent(): + state = State() + getHeads(state) + getTags(state) + getReachableRefs(state) + + return state + +Tmps = [] + +def makeTmp(): + global Tmps + + (fd, fname) = tempfile.mkstemp(prefix="%s-" % Name, suffix=".tmp") + Tmps += [fname] + + return (os.fdopen(fd, "w"), fname) + +def deleteTmps(): + for tmp in Tmps: + os.unlink(tmp) + +def mailTag(key, value): + return "%-11s: %s" % (key, value) + +def generateMailHeader(subject): + + repo = Config.repouri + + if not repo: + + if gitolite: + # Gitolite version. + repo = "ssh://%s@%s/%s" % (whoami, Config.hostname, os.path.basename(os.getcwd())) + else: + # Standard version. + repo = "ssh://%s/%s" % (Config.hostname, os.path.basename(os.getcwd())) + + if repo.endswith(".git"): + repo = repo[0:-4] + + (out, fname) = makeTmp() + print >>out, """From: %s +To: %s +Subject: %s %s +X-Git-Repository: %s +X-Mailer: %s %s + +%s + +""" % (Config.sender, Config.mailinglist, Config.emailprefix, subject, repo, + Name, Version, mailTag("Repository", repo)), + + return (out, fname) + +def sendMail(out, fname): + out.close() + + if Config.debug: + for line in open(fname): + print " |", line, + print "" + else: + stdin = subprocess.Popen("/usr/sbin/sendmail -t", shell=True, stdin=subprocess.PIPE).stdin + for line in open(fname): + print >>stdin, line, + stdin.close() + + # Wait a bit in case we're going to send more mails. Otherwise, the mails + # get sent back-to-back and are likely to end up with identical timestamps, + # which may then make them appear to have arrived in the wrong order. + if not Config.debug: + time.sleep(2) + +def entryAdded(key, value, rev): + log("New %s %s" % (key, value)) + + (out, fname) = generateMailHeader("%s '%s' created" % (key, value)) + + print >>out, mailTag("New %s" % key, value) + print >>out, mailTag("Referencing", rev) + + sendMail(out, fname) + +def entryDeleted(key, value): + log("Deleted %s %s" % (key, value)) + + (out, fname) = generateMailHeader("%s '%s' deleted" % (key, value)) + + print >>out, mailTag("Deleted %s" % key, value) + + sendMail(out, fname) + +# Sends a mail for a notification consistent of two parts: (1) the output of a +# show command, and (2) the output of a diff command. +def sendChangeMail(subject, heads, show_cmd, diff_cmd): + + (out, fname) = generateMailHeader(subject) + + multi = "es" if len(heads) > 1 else "" + heads = ",".join(heads) + + print >>out, mailTag("On branch%s" % multi, heads) + + if Config.link: + url = Config.link.replace("%s", rev) + print >>out, "" + print >>out, mailTag("Link", url) + + footer = "" + show = git(show_cmd) + + for line in show: + if NoDiff in line: + break + + if NoMail in line: + return + + else: + (tmp, tname) = makeTmp() + diff = git(diff_cmd, stdout_to=tmp) + tmp.close() + + size = os.path.getsize(tname) + + if size > Config.maxdiffsize: + footer = "\nDiff suppressed because of size. To see it, use:\n\n git %s" % diff_cmd + tname = None + + print >>out, Separator + + for line in git(show_cmd, all=True): + if line == "---": + print >>out, Separator + else: + print >>out, line + + print >>out, Separator + + if tname: + for line in open(tname): + print >>out, line, + + print >>out, footer + + if Config.debug: + print >>out, "-- " + print >>out, "debug: show_cmd = git %s" % show_cmd + print >>out, "debug: diff_cmd = git %s" % diff_cmd + + sendMail(out, fname) + +# Sends notification for a specific revision. +def commit(current, rev): + if rev in current.reported: + # Already reported in this run of the script. + log("New revision %s, but already reported this time" % rev) + + log("New revision %s" % rev) + current.reported.add(rev) + + heads = [head.split()[-1] for head in git("branch --contains=%s" % rev)] + + if len(set(heads) - Config.allchanges) == 0: + # We have reported full diffs for all already. + return + + subject = git("show '--pretty=format:%%s (%%h)' -s %s" % rev) + subject = "%s: %s" % (",".join(heads), subject[0]) + + show_cmd = "show -s --no-color --find-copies-harder --pretty=medium %s" % rev + diff_cmd = "diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^%s~1 %s " % (rev, rev) + + sendChangeMail(subject, heads, show_cmd, diff_cmd) + +# Sends a diff between two revisions. +def diff(head, first, last): + log("Diffing %s..%s" % (first, last)) + + subject = git("show '--pretty=format:%%s (%%h)' -s %s" % last) + subject = "%s diff: %s" % (head, subject[0]) + + heads = [head] + + show_cmd = "show -s --no-color --find-copies-harder --pretty=medium %s" % last + diff_cmd = "diff --patch-with-stat -m --no-color --find-copies-harder --ignore-space-at-eol %s %s" % (first, last) + + sendChangeMail(subject, heads, show_cmd, diff_cmd) + +# Sends pair-wise diffs for a path of revisions. +def diffPath(head, revs): + last = None + + for rev in revs: + if last: + diff(head, last, rev) + last = rev + +# Sends a commit notifications for a set of revisions, independent of whether +# they already have been reported. +def reportPath(current, revs): + for rev in revs: + commit(current, rev) + +# Sends a summary mail for a set of revisions. +def headMoved(head, path): + log("Head moved: %s -> %s" % (head, path[-1])) + + subject = git("show '--pretty=format:%%s (%%h)' -s %s" % path[-1]) + + (out, fname) = generateMailHeader("%s's head updated: %s" % (head, subject[0])) + + print >>out, "Branch '%s' now includes:" % head + print >>out, "" + + for rev in path: + print >>out, " ", git("show -s --pretty=oneline --abbrev-commit %s" % rev)[0] + + sendMail(out, fname) + +Config = GitConfig(sys.argv[1:]) + +log("Running for %s" % os.getcwd()) + +if Config.debug: + for (name, arg, default, help) in Options: + print >>sys.stderr, "[Option %s: %s]" % (name, Config.__dict__[name]) + +cache = State() + +if os.path.exists(CacheFile): + cache.readFrom(CacheFile) + report = (not Config.updateonly) +else: + log("Initial run. Not generating any mails, just recording current state.") + report = False + +current = getCurrent() + +if Config.diff: + # Manual diff mode. The argument must be of the form "[old-rev..]new-rev". + path = [rev.strip() for rev in Config.diff.split("..")] + if len(path) == 1: + path = ("%s~2" % path[0], path[0]) # sic! ~2. + else: + path = ("%s~1" % path[0], path[1]) + + revs = git(["rev-list", "--reverse", "--first-parent", "--date-order", path[1], "^%s" % path[0]]) + diffPath("", revs) + + sys.exit(0) + +if Config.manual: + # Manual report mode. The argument must be of the form "[old-rev..]new-rev". + path = [rev.strip() for rev in Config.manual.split("..")] + if len(path) == 1: + path = ("%s~1" % path[0], path[0]) + + revs = git(["rev-list", "--reverse", "--date-order", path[1], "^%s" % path[0]]) + reportPath(current, revs) + + sys.exit(0) + +if report: + # Check for changes to the set of heads. + old = set(cache.heads.keys()) + new = set(current.heads.keys()) + + for head in (new - old): + entryAdded("branch", head, current.heads[head]) + + for head in (old - new): + entryDeleted("branch", head) + + stable_heads = new & old + Config.allchanges = Config.allchanges & stable_heads + + # Check tags. + old = set(cache.tags.keys()) + new = set(current.tags.keys()) + + for tag in (new - old): + entryAdded("tag", tag, current.tags[tag]) + + for tag in (old - new): + entryDeleted("tag", tag) + + # Do complete reports for the heads we want to see everything for. + for head in Config.allchanges: + old_rev = cache.heads[head] + new_rev = current.heads[head] + + revs = git(["rev-list", "--reverse", "--first-parent", "--date-order", new_rev, "^%s~1" % old_rev]) + diffPath(head, revs) + + # Check for unreported commits. + + # Sort updates by time. + def _key(rev): + ts = git("show -s '--pretty=format:%%ct' %s" % rev) + return int(ts[0]) + + old = set(cache.revs) + new = set(current.revs) + + new_revs = new - old + + for rev in sorted(new_revs, key=_key): + commit(current, rev) + + # See if heads have moved to include already reported revisions. + for head in stable_heads: + + if head in Config.allchanges: + # Already done complete diffs. + continue + + old_rev = cache.heads[head] + new_rev = current.heads[head] + + path = git(["rev-list", "--reverse", "--date-order", new_rev, "^%s" % old_rev]) + + if len(set(path) - new_revs): + headMoved(head, path) + +if not Config.noupdate: + current.writeTo(CacheFile) + +deleteTmps() diff --git a/roles/batcave/files/infrastructure.fedoraproject.org.conf b/roles/batcave/files/infrastructure.fedoraproject.org.conf new file mode 100644 index 0000000000..5a4cb4cbf3 --- /dev/null +++ b/roles/batcave/files/infrastructure.fedoraproject.org.conf @@ -0,0 +1,206 @@ + + ServerName infrastructure.fedoraproject.org + ServerAlias infrastructure.stg.fedoraproject.org + ServerAdmin webmaster@fedoraproject.org + TraceEnable Off + + # enable git smart http cloning. + SetEnv GIT_PROJECT_ROOT /srv/web/infra/ + SetEnv GIT_HTTP_EXPORT_ALL + ScriptAliasMatch \ + "(?x)^/infra/(.*/(HEAD | \ + info/refs | \ + objects/(info/[^/]+ | \ + [0-9a-f]{2}/[0-9a-f]{38} | \ + pack/pack-[0-9a-f]{40}\.(pack|idx)) | \ + git-(upload|receive)-pack))$" \ + /usr/libexec/git-core/git-http-backend/$1 + + # + # redirect everyone to use https + # + # We can't do this until virt-install can handle https + +# RewriteEngine on +# RewriteCond %{SERVER_PORT} !^443$ +# RewriteRule ^/(.*) https://%{HTTP_HOST}/$1 [L,R] + +# robots location +Alias /robots.txt /srv/web/robots.txt.lockbox01 + +DocumentRoot /srv/web + + Options Indexes FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + + + + Options Indexes FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + + + + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options Indexes FollowSymLinks + AllowOverride None + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options Indexes FollowSymLinks + Order allow,deny + Allow from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options FollowSymLinks + Order allow,deny + Allow from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + +# Needed for cgit cgi +ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + + + ServerName infrastructure.fedoraproject.org + ServerAlias infrastructure.stg.fedoraproject.org + ServerAdmin webmaster@fedoraproject.org + + # enable git smart http cloning. + SetEnv GIT_PROJECT_ROOT /srv/web/infra/ + SetEnv GIT_HTTP_EXPORT_ALL + ScriptAliasMatch \ + "(?x)^/infra/(.*/(HEAD | \ + info/refs | \ + objects/(info/[^/]+ | \ + [0-9a-f]{2}/[0-9a-f]{38} | \ + pack/pack-[0-9a-f]{40}\.(pack|idx)) | \ + git-(upload|receive)-pack))$" \ + /usr/libexec/git-core/git-http-backend/$1 + + SSLEngine on + SSLCertificateFile /etc/pki/tls/certs/wildcard-2014.fedoraproject.org.cert + SSLCertificateKeyFile /etc/pki/tls/private/wildcard-2014.fedoraproject.org.key + SSLCertificateChainFile /etc/pki/tls/certs/wildcard-2014.fedoraproject.org.intermediate.cert + + Header always add Strict-Transport-Security "max-age=15768000; includeSubDomains; preload" + + SSLHonorCipherOrder On + + # https://fedorahosted.org/fedora-infrastructure/ticket/4101#comment:14 + # If you change the protocols or cipher suites, you should probably update + # modules/squid/files/squid.conf-el6 too, to keep it in sync. + SSLProtocol -All +TLSv1 +TLSv1.1 +TLSv1.2 + SSLCipherSuite ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK + +# robots location +Alias /robots.txt /srv/web/robots.txt.lockbox01 + +DocumentRoot /srv/web + + Options Indexes FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + + + + Options Indexes FollowSymLinks + AllowOverride None + Order allow,deny + Allow from all + + + + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options Indexes FollowSymLinks + AllowOverride None + Order deny,allow + Deny from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options Indexes FollowSymLinks + Order allow,deny + Allow from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + + + Options FollowSymLinks + Order allow,deny + Allow from all + Include "conf.d/infrastructure.fedoraproject.org/allows" + + +# Needed for cgit cgi +ScriptAlias /cgi-bin/ "/var/www/cgi-bin/" + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + + AllowOverride None + Options None + Order allow,deny + Allow from all + + diff --git a/roles/batcave/files/mime-types.conf b/roles/batcave/files/mime-types.conf new file mode 100644 index 0000000000..9fed4cdee2 --- /dev/null +++ b/roles/batcave/files/mime-types.conf @@ -0,0 +1 @@ +AddType text/plain .rst \ No newline at end of file diff --git a/roles/batcave/files/oshift_mod.py b/roles/batcave/files/oshift_mod.py new file mode 100644 index 0000000000..592908bc5e --- /dev/null +++ b/roles/batcave/files/oshift_mod.py @@ -0,0 +1,742 @@ +#!/usr/bin/env python +from __future__ import print_function + +""" +This is a python interface for using Openshift-2.0 REST +version = 2.0 changed the basic support to use the new requests module + (http://docs.python-requests.org/en/latest/index.html) + +Source: https://github.com/openshift/python-interface/raw/master/oshift/__init__.py + +""" + +import os +import sys +import logging +from optparse import OptionParser +import time +import traceback +import json + +import base64 +import requests + + +class OpenShiftException(BaseException): + pass + + +class OpenShiftLoginException(OpenShiftException): + """Authorization failed.""" + pass + +class OpenShiftAppException(OpenShiftException): + """App not found.""" + pass + +class OpenShiftNullDomainException(OpenShiftException): + """User's domain hasn't been initialized.""" + pass + + +class OpenShift500Exception(OpenShiftException): + """Internal Server Error""" + pass + +#### set this to True if we want to enable performance analysis +DOING_PERFORMANCE_ANALYSIS = False + +global log + + +def config_logger(): + # create formatter + formatter = logging.Formatter("%(levelname)s [%(asctime)s] %(message)s", + "%H:%M:%S") + logger = logging.getLogger("dump_logs") + log_formatter = logging.Formatter( + "%(name)s: %(asctime)s - %(levelname)s: %(message)s") + + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) + logger.addHandler(stream_handler) + return logger + + +def config_parser(): + # these are required options. + parser.set_defaults(VERBOSE=False) + parser.set_defaults(DEBUG=False) + parser.add_option("-d", action="store_true", dest="DEBUG", help="enable DEBUG (default true)") + parser.add_option("-i", "--ip", default="openshift.redhat.com", help="ip addaress of your devenv") + parser.add_option("-v", action="store_true", dest="VERBOSE", help="enable VERBOSE printing") + parser.add_option("-u", "--user", default=None, help="User name") + parser.add_option("-p", "--password", default=None, help="RHT password") + (options, args) = parser.parse_args() + + if options.user is None: + options.user = os.getenv('OPENSHIFT_user_email') + + if options.password is None: + options.password = os.getenv('OPENSHIFT_user_passwd') + + return options, args +log = config_logger() +parser = OptionParser() + + +# helper function for to measure timedelta. +def timeit(method): + + def timed(*args, **kw): + ts = time.time() + result = method(*args, **kw) + te = time.time() + + log.info("%r (%r, %r) %2.2f sec" % (method.__name__, args, kw, te-ts)) + return result, te-ts + + return timed + + +class conditional_decorator(object): + def __init__(self, dec, condition): + self.decorator = dec + self.condition = condition + + def __call__(self, func): + if not self.condition: + return func + else: + return self.decorator(func) + + +class RestApi(object): + """ + A base connection class to derive from. + """ + + proto = 'https' + host = '127.0.0.1' + port = 443 + username = None + password = None + headers = None + response = None + base_uri = None + verbose = False + debug = False + + def __init__(self, host=None, port=443, username=username, password=password, + debug=False, verbose=False, proto=None, headers=None): + if proto is not None: + self.proto = proto + + if host is not None: + self.host = host + + if username: + self.username = username + + if password: + self.password = password + + if headers: + self.headers = headers + + if verbose: + self.verbose = verbose + + self.debug = debug + self.base_uri = self.proto + "://" + host + "/broker/rest" + + + def _get_auth_headers(self, username=None, password=None): + if username: + self.username = username + + if password: + self.password = password + + return (self.username, self.password) + + def request(self, url, method, headers=None, params=None): + """ + wrapper method for Requests' methods + """ + if url.startswith("https://") or url.startswith("http://"): + self.url = url # self.base_uri + url + else: + self.url = self.base_uri + url + log.debug("URL: %s" % self.url) + auth = (self.username, self.password) # self._get_auth_headers() + #auth = self._get_auth_headers() + _headers = self.headers or {} + if headers: + _headers.update(headers) + if 'OPENSHIFT_REST_API' in os.environ: + user_specified_api_version = os.environ['OPENSHIFT_REST_API'] + api_version = "application/json;version=%s" % user_specified_api_version + + _headers['Accept'] = api_version + + self.response = requests.request( + auth=None if None in auth else auth, + method=method, url=self.url, params=params, + headers=_headers, timeout=130, verify=False + ) + + try: + raw_response = self.response.raw + except Exception as e: + print("-"*80, file=sys.stderr) + traceback.print_exc(file=sys.stderr) + print("-"*80, file=sys.stderr) + raise e + + self.data = self.response.json() + # see https://github.com/kennethreitz/requests/blob/master/requests/status_codes.py + if self.response.status_code == requests.codes.internal_server_error: + raise OpenShift500Exception('Internal Server Error: %s' % self.data) + + if self.response.status_code == (200 or 201): + print("-"*80, file=sys.stderr) + log.debug("status: %s" % self.response.status_code) + #log.debug("msg: %s" % self.data()['messages'][0]['text']) + # the raw_response is not available + #log.debug("raw: %s"%raw_response) + print("-"*80, file=sys.stderr) + return (self.response.status_code, self.data) + + +class Openshift(object): + """ + wrappers class around REST API so use can use it with python + """ + rest = None + user = None + passwd = None + + def __init__(self, host, user=None, passwd=None, debug=False, verbose=False, logger=None, proto=None, headers=None): + if user: + self.user = user + if passwd: + self.passwd = passwd + if logger: + global log + log = logger + self.rest = RestApi(host=host, username=self.user, password=self.passwd, debug=debug, verbose=verbose, proto=proto, headers=headers) + if 'OPENSHIFT_REST_API' in os.environ: + self.REST_API_VERSION = float(os.environ['OPENSHIFT_REST_API']) + else: + # just get the latest version returned from the Server + api_version, api_version_list = self.api_version() + self.REST_API_VERSION = api_version + + def get_href(self, top_level_url, target_link, domain_name=None): + status, res = self.rest.request(method='GET', url=top_level_url) + index = target_link.upper() + if status == 'Authorization Required': + #log.error("Authorization failed. (Check your credentials)") + raise OpenShiftLoginException('Authorization Required') + + if domain_name is None: + if self.rest.response.json()['data']: + res = self.rest.response.json()['data'][0]['links'][index] + + return (res['href'], res['method']) + else: + raise OpenShiftNullDomainException("No domain has been initialized.") + #return ('Not Found', self.rest.response.json) + + else: # domain name is specified, now find a match + json_data = self.rest.response.json()['data'] + if json_data: + for jd in json_data: + if jd['name'] == domain_name: + res = jd['links'][index] + return (res['href'], res['method']) + ### if here, then user has given a domain name that does not match what's registered with the system + return("Not Found", None) + else: + return(None, None) + + ##### /user (sshkey) + #@conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def get_user(self): + log.debug("Getting user information...") + (status, raw_response) = self.rest.request(method='GET', url='/user') + + if status == 'OK': + return (status, self.rest.response.json()['data']['login']) + else: + return (status, raw_response) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def keys_list(self): + log.debug("Getting ssh key information...") + (status, raw_response) = self.rest.request(method='GET', url='/user/keys') + return (status, raw_response) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def key_add(self, kwargs): + """ + params: {name, type, key_path} + """ + ssh_key_str = None + if 'key_str' in kwargs: + ssh_key_str = kwargs['key_str'] + else: + if 'key' not in kwargs: + # use a default path + sshkey = '~/.ssh/id_rsa.pub' + else: + sshkey = kwargs['key'] + ssh_path = os.path.expanduser(sshkey) + ssh_key_str = open(ssh_path, 'r').read().split(' ')[1] + + if 'name' not in kwargs: + + kwargs['name'] = 'default' + + if 'type' not in kwargs: + kwargs['type'] = 'ssh-rsa' + + data_dict = { + 'name': kwargs['name'], + 'type': kwargs['type'], + 'content': ssh_key_str + } + + params = data_dict + status, raw_response = self.rest.request(method='POST', url='/user/keys', params=params) + return (status, raw_response) + + ##### /domains + #@conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + # TODO: should the rhlogin really be hardcoded in this function? + def domain_create(self, name, rhlogin='nate@appsembler.com'): + log.debug("Creating domain '%s'" % name) + params = { + 'id': name, + 'rhlogin': rhlogin + } + + status, res = self.rest.request(method='POST', url='/domains', params=params) + return (status, res) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def domain_delete(self, domain_name=None, force=True): + """ destroy a user's domain, if no name is given, figure it out""" + if domain_name is None: + status, domain_name = self.domain_get() + + url, method = self.get_href('/domains', 'delete', domain_name) + log.info("URL: %s" % url) + #res = self.rest.response.data[0]['links']['DELETE'] + if force: + params = {'force': 'true'} + if url: + return self.rest.request(method=method, url=url, params=params) + else: # problem + return (url, self.rest.response.raw) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def domain_get(self, name=None): + log.info("Getting domain information...") + url, method = self.get_href('/domains', 'get', name) + if url == 'Not Found': + return ('Not Found', None) + else: + (status, raw_response) = self.rest.request(method=method, url=url) + if status == 200: + if self.REST_API_VERSION < 1.6: + domain_index_name = 'id' + else: + domain_index_name = 'name' + return (status, self.rest.response.json()['data'][domain_index_name]) + + def domain_update(self, new_name): + params = {'id': new_name} + url, method = self.get_href("/domains", 'update') + (status, res) = self.rest.request(method=method, url=url, params=params) + return (status, res) + + def app_list(self): + url, method = self.get_href('/domains', 'list_applications') + (status, res) = self.rest.request(method=method, url=url) + return (status, self.rest.response.json()['data']) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_create(self, app_name, app_type, scale='false', init_git_url=None): + url, method = self.get_href('/domains', 'add_application') + valid_options = self.rest.response.json()['data'][0]['links']['ADD_APPLICATION']['optional_params'][0]['valid_options'] + #if app_type not in valid_options: + # log.error("The app type you specified '%s' is not supported!" % app_type) + # log.debug("supported apps types are: %s" % valid_options) + + try: + json_data = json.loads(json.dumps(app_type)) + except: + json_data = None + + if json_data: + # translate json data into list + is_dict = all(isinstance(i, dict) for i in json_data) + cart_info = [] + + if is_dict: + # need to construct a cart as a list from dictionary + for data in json_data: + cart_info.append(data['name']) + else: + cart_info = json_data + + else: + cart_info.append(app_type) + + data_dict = { + 'name': app_name, + 'cartridges[]': cart_info, + 'scale': scale, + } + + if init_git_url: + data_dict['initial_git_url'] = init_git_url + + params = data_dict + #log.debug("URL: %s, METHOD: %s" % (url, method)) + (status, res) = self.rest.request(method=method, url=url, params=params) + return (status, res) + + ##### /cartridges + def cartridges(self): + (status, raw_response) = self.rest.request(method='GET', url='/cartridges') + if status == 'OK': + # return a list of cartridges that are supported + return (status, self.rest.response.json()['data']) + else: + return (status, raw_response) + + ##### /api get a list of support operations + def api(self): + #log.debug("Getting supported APIs...") + (status, raw_response) = self.rest.request(method='GET', url='/api') + return (status, raw_response) + + def api_version(self): + # return the current version being used and the list of supported versions + status, res = self.api() + return (float(res['version']), res['supported_api_versions']) + + ##### helper functions + def do_action(self, kwargs): + op = kwargs['op_type'] + if op == 'cartridge': + status, res = self.cartridge_list(kwargs['app_name']) + elif op == 'keys': + status, res = self.keys_list() + + json_data = self.rest.response.json() + action = kwargs['action'] + name = kwargs['name'] + raw_response = None + for data in json_data['data']: + if data['name'] == name: + params = data['links'][action] + log.debug("Action: %s" % action) + if len(params['required_params']) > 0: + # construct require parameter dictionary + data = {} + for rp in params['required_params']: + param_name = rp['name'] + if kwargs['op_type'] == 'cartridge': + data[param_name] = action.lower() + else: + data[param_name] = kwargs[param_name] + data = data + else: + data = None + (status, raw_response) = self.rest.request(method=params['method'], + url=params['href'], + params=data) + return (status, self.rest.response.json()) + + return (status, raw_response) + + #### application tempalte + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_templates(self): + (status, raw_response) = self.rest.request(method='GET', url='/application_template') + if status == 'OK': + return (status, self.rest.response.json()) + else: + return (status, raw_response) + + ##### keys + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def key_delete(self, key_name): + """ + li.key_delete('ssh_key_name') + + """ + params = {"action": 'DELETE', 'name': key_name, "op_type": 'keys'} + return self.do_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def key_update(self, kwargs): # key_name, key_path, key_type='ssh-rsa'): + """ + li.key_update({'name': 'new_key_name', 'key': new_key_path}) + + """ + key_path = kwargs['key'] + key_name = kwargs['name'] + if 'key_type' in kwargs: + key_type = kwargs['key_type'] + else: + key_type = 'ssh-rsa' + ssh_path = os.path.expanduser(key_path) + ssh_key_str = open(ssh_path, 'r').read().split(' ')[1] + + params = {'op_type': 'keys', 'action': 'UPDATE', 'name': key_name, 'content': ssh_key_str, 'type': key_type} + return self.do_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def key_get(self, name): + """ + li.key_get('target_key_name') + returns the actual key content :$ + """ + params = {'action': 'GET', 'name': name, 'op_type': 'keys'} + url = "/user/keys/" + name + (status, raw_response) = self.rest.request(method='GET', url=url) + if status == 'OK': + return status, self.rest.response.json()['data'] + else: + return (status, raw_response) + + def key_action(self, kwargs): + status, res = self.keys_list() + json_data = self.rest.response.json() + action = kwargs['action'] + name = kwargs['name'] + for data in json_data['data']: + if data['name'] == name: + params = data['links'][action] + log.debug("Action: %s" % action) + if len(params['required_params']) > 0: + # construct require parameter dictionary + data = {} + for rp in params['required_params']: + param_name = rp['name'] + data[param_name] = kwargs[param_name] + data = data + else: + data = None + break + (status, raw_response) = self.rest.request(method=params['method'], + url=params['href'], + params=data) + return (status, raw_response) + + ##### apps + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_create_scale(self, app_name, app_type, scale, init_git_url=None): + self.app_create(app_name=app_name, app_type=app_type, scale=scale, init_git_url=init_git_url) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_delete(self, app_name): + params = {'action': 'DELETE', 'app_name': app_name} + return self.app_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_start(self, app_name): + params = {"action": 'START', 'app_name': app_name} + return self.app_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_stop(self, app_name): + params = {"action": 'STOP', 'app_name': app_name} + return self.app_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_restart(self, app_name): + params = {"action": 'RESTART', 'app_name': app_name} + return self.app_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_force_stop(self, app_name): + params = {"action": 'FORCE_STOP', 'app_name': app_name} + return self.app_action(params) + + @conditional_decorator(timeit, DOING_PERFORMANCE_ANALYSIS) + def app_get_descriptor(self, app_name): + params = {'action': 'GET', 'app_name': app_name} + return self.app_action(params) + + ############################################################# + # event related functions + ############################################################# + def app_scale_up(self, app_name): + params = {'action': 'SCALE_UP', 'app_name': app_name} + return self.app_action(params) + + def app_scale_down(self, app_name): + params = {'action': 'SCALE_DOWN', 'app_name': app_name} + return self.app_action(params) + + def app_add_alias(self, app_name, alias): + params = {'action': 'ADD_ALIAS', 'app_name': app_name, 'alias': alias} + return self.app_action(params) + + def app_remove_alias(self, app_name, alias): + params = {'action': 'REMOVE_ALIAS', 'app_name': app_name, 'alias': alias} + return self.app_action(params) + + def app_get_estimates(self): + url, method = self.get_href('/estimates', 'get_estimate') + (status, res) = self.rest.request(method=method, url=url) + return (status, self.rest.response.json()['data']) + + #params = {'action': 'GET_ESTIMATE'} + #return self.app_action(params) + + def app_action(self, params): + """ generic helper function that is capable of doing all the operations + for application + """ + # step1. find th url and method + status, res = self.app_list() + + app_found = False + action = params['action'] + if 'app_name' in params: + app_name = params['app_name'] + if 'cartridge' in params: + cart_name = params['cartridge'] + + for app in res: + #for app in res['data']: + + if app['name'] == app_name: + # found match, now do your stuff + params_dict = app['links'][action] + method = params_dict['method'] + log.info("Action: %s" % action) + data = {} + if len(params_dict['required_params']) > 0: + param_name = params_dict['required_params'][0]['name'] + rp = params_dict['required_params'][0] + #data[param_name] = cart_name #'name'] = rp['name'] + for rp in params_dict['required_params']: + # construct the data + param_name = rp['name'] + if param_name == 'event': + if isinstance(rp['valid_options'], list): + data[param_name] = rp['valid_options'][0] + else: + data[param_name] = rp['valid_options'] + else: + data[param_name] = params[param_name] # cart_name #params['op_type'] + #data[param_name] = params[param_name] + data = data + else: + data = None + req_url = params_dict['href'] + (status, raw_response) = self.rest.request(method=method, url=req_url, params=data) + app_found = True + return (status, raw_response) + if not app_found: + raise OpenShiftAppException("Can not find the app matching your request") + #log.error("Can not find app matching your request '%s'" % app_name) + #return ("Error", None) + + def get_gears(self, app_name, domain_name=None): + """ return gears information """ + params = {"action": 'GET_GEAR_GROUPS', 'app_name': app_name} + return self.app_action(params) + + ################################ + # cartridges + ################################ + def cartridge_list(self, app_name): + params = {"action": 'LIST_CARTRIDGES', 'app_name': app_name} + return self.app_action(params) + + def cartridge_add(self, app_name, cartridge): + params = {"action": 'ADD_CARTRIDGE', 'app_name': app_name, + 'cartridge': cartridge} + status, res = self.app_action(params) + return (status, self.rest.response.json()['messages']) + + def cartridge_delete(self, app_name, name): + params = {"action": 'DELETE', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def cartridge_start(self, app_name, name): + params = {"action": 'START', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def cartridge_stop(self, app_name, name): + params = {"action": 'STOP', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def cartridge_restart(self, app_name, name): + params = {"action": 'RESTART', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def cartridge_reload(self, app_name, name): + params = {"action": 'RELOAD', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def cartridge_get(self, app_name, name): + params = {"action": 'GET', 'name': name, "op_type": 'cartridge', 'app_name': app_name} + return self.do_action(params) + + def app_template_get(self): + """ return a list of application template from an app """ + status, res = self.rest.request(method='GET', url='/application_template') + if status == 'OK': + return (status, self.rest.response.json()['data']) + else: + return (status, res) + + +def sortedDict(adict): + keys = list(adict.keys()) + keys.sort() + return map(adict.get, keys) + + +def perf_test(li): + cart_types = ['php-5.3'] + od = { + 1: {'name': 'app_create', 'params': {'app_name': 'perftest'}}, + #2: {'name': 'app_delete', 'params': {'app_name': 'perftest'}}, + } + sod = sortedDict(od) + #li.domain_create('blahblah') + cart_types = ['php-5.3'] # 'php-5.3', 'ruby-1.8', 'jbossas-7'] + for cart in cart_types: + for action in sod: + method_call = getattr(li, action['name']) + k, v = list(action['params'].items()[0]) + if action['name'] == 'app_create': + method_call(v, cart) + else: + method_call(v) + + +if __name__ == '__main__': + (options, args) = config_parser() + li = Openshift(host=options.ip, user=options.user, passwd=options.password, + debug=options.DEBUG,verbose=options.VERBOSE) + status, res = li.domain_get() + self.info('xxx', 1) + #status, res = li.app_create(app_name="app1", app_type=["ruby-1.8", "mysql-5.1"], init_git_url="https://github.com/openshift/wordpress-example") + #status, res = li.app_create(app_name="app2", app_type="php-5.3", init_git_url="https://github.com/openshift/wordpress-example") + #status, res = li.app_create(app_name="app3", app_type=[{"name": "ruby-1.8"}, {"name": "mysql-5.1"}], init_git_url="https://github.com/openshift/wordpress-example") + diff --git a/roles/batcave/files/retrieve-security-question.py b/roles/batcave/files/retrieve-security-question.py new file mode 100755 index 0000000000..a75e6519e4 --- /dev/null +++ b/roles/batcave/files/retrieve-security-question.py @@ -0,0 +1,108 @@ +#!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# Use this script to retrieve the security_question and security_answer from FAS (requires FAS >= 0.8.14) +# Author: Patrick Uiterwijk +# +# Copyright 2012 Patrick Uiterwijk. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE FEDORA PROJECT ''AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +# EVENT SHALL THE FREEBSD PROJECT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the Fedora Project. + + +import os +import getpass +import sys +import gpgme +from fedora.client import AccountSystem +from fedora.client import AuthError +from fedora.client import ServerError +import argparse +from io import BytesIO + + +parser = argparse.ArgumentParser() +parser.add_argument('admin_user', help='The user as which to log in to retrieve the question and answer') +parser.add_argument('target_user', help='The user of which to retrieve the security question and answer') +parser.add_argument('--verbose', action='store_true') +parser.add_argument('--no-answer', action='store_true', help='Only show the question, do not decrypt the answer') +parser.add_argument('--site', help='The FAS URL to get the information from') +parser.add_argument('--insecure', action='store_true', default=False, + help='Do not check the certificate for the server. *WARNING*: Only use this for testing') +parser.add_argument('--gpg_home', help='The directory where secring.gpg and pubring.gpg reside') +args = parser.parse_args() + +args.admin_pass = getpass.getpass() + +if args.site == None: + args.site = 'https://admin.fedoraproject.org/accounts/' + +if args.verbose: + print 'Using site: %(site)s' % {'site': args.site} + +if args.verbose: + if args.gpg_home == None: + print 'Using default gpg_home' + else: + print 'Using gpg_home: %(gpghome)s' % {'gpghome': args.gpg_home} + +if args.gpg_home != None: + os.putenv('GNUPGHOME', args.gpg_home) + +fas = AccountSystem(args.site, username=args.admin_user, password=args.admin_pass, insecure=args.insecure) + +if args.verbose: + print 'Getting user details...' +try: + details = fas.person_by_username(args.target_user) +except AuthError: + print 'Failed to login to FAS. Please check admin_user and admin_pass!' + sys.exit(2) +except ServerError: + print 'Failed to retrieve user details: the server reported an error!' + sys.exit(3) + +if not 'username' in details.keys(): + print 'Error: user %(username)s is not known on this FAS site!' % {'username': args.target_user} + sys.exit(4) + +if not 'security_question' in details.keys(): + print 'Error: security_question was not retrieved by FAS! Are you sure you are using FAS >= 0.8.14, and that admin_user has the privileges to retrieve security_question?' + sys.exit(5) + +if details.security_question == None or details.security_answer == None: + print 'Error: unable to retrieve security_question or security_answer. Are you sure you have privileges to return this information?' + sys.exit(6) + +if not args.no_answer: + if args.verbose: + print 'Decrypting answer...' + cipher = BytesIO(details.security_answer.encode('utf-8')) + plain = BytesIO() + ctx = gpgme.Context() + ctx.decrypt(cipher, plain) + details.security_answer = plain.getvalue() + +print 'Security question: %(question)s' % {'question': details.security_question} +if not args.no_answer: + print 'Security answer: %(answer)s' % {'answer': details.security_answer} diff --git a/roles/batcave/files/sync-openshift-keys.py b/roles/batcave/files/sync-openshift-keys.py new file mode 100755 index 0000000000..182ce94332 --- /dev/null +++ b/roles/batcave/files/sync-openshift-keys.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# Copyright 2012 Patrick Uiterwijk +# This file has been released as LGPLv3+, see COPYING for the complete license +import sys +import string +import ConfigParser +from argparse import ArgumentParser +from fedora.client import AccountSystem +from oshift_mod import Openshift +from getpass import getpass + +def get_keys(host,user,passwd): + os = Openshift(host=host,user=user,passwd=passwd) + (resp, content) = os.keys_list() + if resp != 200: + print('ERROR! Result: %(resp)s' % {'resp': resp}) + sys.exit(1) + return os.rest.response.json()['data'] + +def add_key(host,user,passwd,key_name,key_type,key_contents, verbose=False): + if verbose: + print('Adding key %(keyname)s' % {'keyname': key_name}) + os = Openshift(host=host,user=user,passwd=passwd) + (resp, content) = os.key_add(name=key_name, type=key_type, key_str=key_contents) + if resp != 200: + print('ERROR! Result: %(resp)s' % {'resp': resp}) + sys.exit(2) + if verbose: + print('Done') + return os.rest.response.json()['data'] + +def remove_key(host,user,passwd,key_name, verbose=False): + if verbose: + print('Removing key %(keyname)s' % {'keyname': key_name}) + os = Openshift(host=host,user=user,passwd=passwd) + (resp, content) = os.key_delete(key_name) + if resp != 200: + print 'ERROR! Result: %(resp)s' % {'resp': resp} + sys.exit(3) + if verbose: + print('Done') + return os.rest.response.json()['data'] + +def get_users_to_have_access(fas, groups): + all_users = set() + for group in groups: + new_users = fas.group_members(group) + for new_user in new_users: + all_users.add(new_user['username']) + return all_users + +def get_users_ssh_keys(fas, users): + keys = {} + user_data = fas.user_data() + for userid in user_data.keys(): + if user_data[userid]['username'] in users: + if user_data[userid]['ssh_key']: + contents = user_data[userid]['ssh_key'] + if contents.split(' ') > 1: + key_type = contents.split(' ')[0] + key_contents = contents.split(' ')[1] + keys[user_data[userid]['username']] = {'type': key_type, + 'contents': key_contents, + 'username': user_data[userid]['username']} + return keys + +def get_keys_to_remove(keys_in_openshift, keys_in_fas): + keys_to_remove = set() + for key in keys_in_openshift: + keys_to_remove.add(key['name']) + for key_in_fas in keys_in_fas: + if keys_in_fas[key_in_fas]['contents'] == key['content']: + keys_to_remove.remove(key['name']) + return keys_to_remove + +def get_keys_to_add(keys_in_openshift, keys_in_fas): + usernames_to_add = set() + for username in keys_in_fas: + usernames_to_add.add(username) + for key in keys_in_openshift: + if key['content'] == keys_in_fas[username]['contents']: + usernames_to_add.remove(username) + keys_to_add = [] + for username in usernames_to_add: + keys_to_add.append(keys_in_fas[username]) + return keys_to_add + +def remove_keys(openshift_host, openshift_user, openshift_pass, to_remove, verbose=False): + if verbose: + print('Removing the following keys:') + print(to_remove) + for key in to_remove: + remove_key(openshift_host, openshift_user, openshift_pass, key, verbose=verbose) + if verbose: + print('Done') + +def add_keys(openshift_host, openshift_user, openshift_pass, to_add, prefix, verbose=False): + if verbose: + print('Adding the following keys:') + print(to_add) + for key in to_add: + add_key(openshift_host, openshift_user, openshift_pass, '%(prefix)s%(username)s' % {'prefix': prefix, 'username': key['username']}, key['type'], key['contents'], verbose=verbose) + if verbose: + print('Done') + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument('-config_file', help='The configuration file to use', default='/etc/sync-openshift-keys.conf') + parser.add_argument('--verbose', '-v', help='Make the script more verbose', action='store_true') + args = parser.parse_args() + config = ConfigParser.ConfigParser() + config.read(args.config_file) + fas = AccountSystem(config.get('fas', 'url'), username=config.get('fas', 'user'), password=config.get('fas', 'pass')) + fas.insecure = True + if args.verbose: + print('Getting users...') + users = get_users_to_have_access(fas, string.split(config.get('general', 'groups'), ',')) + if args.verbose: + print('Done: %s' % users) + print('Getting keys in FAS...') + keys_fas = get_users_ssh_keys(fas, users) + if args.verbose: + print('Done: %s') + print('Getting keys in Openshift...') + keys_openshift = get_keys(config.get('openshift', 'host'), config.get('openshift', 'user'), config.get('openshift', 'pass')) + if args.verbose: + print('Done') + print('Getting keys to remove...') + keys_to_remove = get_keys_to_remove(keys_openshift, keys_fas) + if args.verbose: + print('Done') + print('Getting keys to add...') + keys_to_add = get_keys_to_add(keys_openshift, keys_fas) + if args.verbose: + print('Done') + remove_keys(config.get('openshift', 'host'), config.get('openshift', 'user'), config.get('openshift', 'pass'), keys_to_remove, verbose=args.verbose) + add_keys(config.get('openshift', 'host'), config.get('openshift', 'user'), config.get('openshift', 'pass'), keys_to_add, config.get('general', 'keyname_prefix'), verbose=args.verbose) diff --git a/roles/batcave/files/zodbot-announce-commits.py b/roles/batcave/files/zodbot-announce-commits.py new file mode 100644 index 0000000000..0a0ea99d5f --- /dev/null +++ b/roles/batcave/files/zodbot-announce-commits.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# encoding: utf-8 +# (c) 2012 Red Hat, Inc. +# Authored by Ricky Elrod +# But when it breaks, don't yell at him because that's mean. +# update hook for FI repos -> zodbot. + +import os +import sys +import subprocess +import shlex +import socket +import urllib + +ZODBOT_SERVER = "value01" +ZODBOT_PORT = 5050 + +hook = sys.argv[0] +repodir = sys.argv[1] +channel = sys.argv[2] +old = sys.argv[3] +new = sys.argv[4] +branch = sys.argv[5] + +# Split on /, nuke empties from the result, use the last nonempty +# element. This lets us not care if there's a trailing slash. +repodir = filter(None, repodir.split('/'))[-1] + + +def run_command(command): + """ Run a command and return a hash with the resulting stdout/stderr.""" + escaped = shlex.split(command) + cmd = subprocess.Popen(escaped, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = cmd.communicate() + return {"stdout": stdout, "stderr": stderr} + + +def construct_url(slug): + """ Return a space-padded url to the commit. + + If and only if it is handled by cgit. Otherwise, return an empty string. + """ + + # Our long url template. + tmpl = "http://infrastructure.fedoraproject.org/cgit/{repo}/commit/?id={slug}" + + repo = repodir + ".git" + + with open('/etc/cgit-projects-lockbox01', 'r') as f: + lines = [line.strip() for line in f.readlines()] + + if repo in lines and slug: + return " " + tmpl.format(repo=repo, slug=slug) + else: + return "" + + +def parse_commit(commit): + """ + So we're given a commit in the form of: + + --- + Ricky Elrod - test-repo:a045150 ---- add some more test files... + + A foobar/asdf/testfile.1 + A foobar/testfile.2 + --- + + Essentially, we rip out the first line and set it aside. + Then all the other lines will begin with M/C/R/A/D/U. + Replace those letters with fancy little symbols (like + for A). + Combine them together in a list/array. + Show the first 4 and if more exist, append '...' to the list. + Lastly, replace the "----" in the original line above with these. + """ + + lines = commit.split("\n") + message = lines.pop(0) + files = [] + + # extract the commit hash from the first line. + slug = None + try: + slug = message.split(' -')[1].strip().split(':')[1] + except IndexError: + print "** Couldn't parse slug from git-rev.", message + + # The remaining lines are files changed. + for changed_file in filter(None, lines): + status, filename = changed_file.split() + if status == "M" or status == "R": + symbol = "*" + elif status == "C" or status == "A": + symbol = "+" + elif status == "D": + symbol = "-" + else: + symbol = "?" + + files.append(symbol + filename) + + # Show the first 4 files changed, and if there are more, add a '...' + # If no files were changed don't show empty [] because it looks tacky. + fileslist = ' '.join(files[0:4]) + if len(files): + fileslist = '[' + fileslist + + if len(files) > 4: + fileslist += ' ...' + + fileslist += ']' + else: + fileslist = '-' + + padded_url = construct_url(slug) + + # Replace the ---- with the files list... + return message.replace('----', fileslist, 1) + padded_url + +# Get a list of commits to report. +if branch == 'master': + revs = run_command("git rev-list ^%s %s" % (old, new))["stdout"].split("\n") + revs = filter(None, revs) + revs.reverse() + + for commit_hash in revs: + # Get the commit in a format that we can deal with + commit = run_command( + "git show --name-status " + commit_hash + " --oneline " + "--format='%an - " + repodir + ":%h ---- %s'") + parsed_commit = parse_commit(commit["stdout"]) + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((ZODBOT_SERVER, ZODBOT_PORT)) + s.sendall(channel+ " " + parsed_commit) + s.close() diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index 4ac451b944..f033f76a86 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -58,6 +58,78 @@ - batcave - config +# +# Scripts +# + +# +# Hook to notify on git commits used in git repos +# + +- name: setup git-notifier script + copy: src=git-notifier dest=/usr/local/bin/git-notifier mode=0755 + tags: + - batcave + - config + +# +# Script used to gather encrypted security questions from fas +# + +- name: setup /usr/local/bin/retrieve-security-question.py + copy: src=retrieve-security-question.py dest=/usr/local/bin/retrieve-security-question.py mode=0755 + tags: + - batcave + - config + +# +# Script to sync ssh keys from fas to openshift instances. +# + +- name: setup python module for openshift sync script + copy: src=oshift_mod.py dest=/usr/local/bin/oshift_mod.py mode=0644 + tags: + - batcave + - config + +- name: setup setup sync-openshift-keys config + template: src=sync-openshift-keys.conf.j2 dest=/etc/sync-openshift-keys.conf mode=0600 + tags: + - batcave + - config + +- name: setup setup sync-openshift-keys script + copy: src=sync-openshift-keys.py dest=/usr/local/bin/sync-openshift-keys.py mode=0755 + tags: + - batcave + - config + +# The zodbot server must allow TCP on whatever port zodbot is listening on +# for this to work (currently TCP port 5050). +# Once that is done, you can symlink /usr/local/bin/zodbot-announce-commits.py +# to the 'hooks' directory of the bare repo you're wishing to receive commits +# for, then add a hooks.zodbotchannel to the repo's config file. +# Lastly, add the following lines to your 'update' hook: +# reposource=$(git config hooks.reposource) +# zodbot_channel=$(git config hooks.zodbotchannel) +# python $reposource/hooks/zodbot-announce-commits.py $reposource $zodbot_channel $oldrev $newrev ${1#refs/heads/} + +- name: install zodbot-announce-commits script + copy: src=zodbot-announce-commits.py dest=/usr/local/bin/zodbot-announce-commits.py mode=0755 + tags: + - batcave + - config + +# +# This is another script to announce commits, this time to the fedmsg bus +# + +- name: install fedmsg-announce-commits script + copy: src=fedmsg-announce-commits.py dest=/usr/local/bin/fedmsg-announce-commits.py mode=0755 + tags: + - batcave + - config + # # This script checks all the virthosts and logs what guests they are running. # @@ -88,26 +160,34 @@ # - batcave # - config +# +# Setup web server config +# +- name: install web server config for batcave (mimetypes) + copy: src=mime-typesconf dest=/etc/httpd/conf.d/mime-types.conf mode=0644 + tags: + - batcave + - config + - httpd + +- name: install web server config for batcave (access rules) + copy: src=allows dest=/etc/httpd/conf.d/allows mode=0644 + tags: + - batcave + - config + - httpd + +- name: install web server config for batcave (main config) + copy: src=infrastructure.fedoraproject.org.conf dest=/etc/httpd/conf.d/infrastructure.fedoraproject.org.conf mode=0644 + tags: + - batcave + - config + - httpd # still to convert from puppet: # include scripts::check-sshkeys -# include scripts::git-notifier -# include scripts::retrieve-security-question -# include scripts::sync-openshift-keys -# include scripts::zodbotAnnounceCommits -# include scripts::fedmsgAnnounceCommits # include scripts::sync-rhn -# # include repo2json # include ansible_utils::ansible_utils -# -# include rsync::server # include scripts::geoip-retriever # include geoip-retriever -# -# httpd::site { "infrastructure.fedoraproject.org": } -# -# httpd::mime-type { "restructured text docs": -# website => "infrastructure.fedoraproject.org", -# mimetype => "text/plain", -# extensions => [ ".rst" ], diff --git a/roles/batcave/templates/sync-openshift-keys.conf.j2 b/roles/batcave/templates/sync-openshift-keys.conf.j2 new file mode 100644 index 0000000000..151ba7d02e --- /dev/null +++ b/roles/batcave/templates/sync-openshift-keys.conf.j2 @@ -0,0 +1,13 @@ +[general] +groups=sysadmin-main,sysadmin-noc +keyname_prefix=fas + +[openshift] +host=openshift.redhat.com +user={{ openshift_status_username }} +pass={{ openshift_status_password }} + +[fas] +url=https://admin.fedoraproject.org/accounts/ +user={{ fedorathirdpartyUser }} +pass={{ fedorathirdpartyPassword }}