From 6e9746864ead8dae86cde7844289afc3901c6d70 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Mon, 9 Jun 2014 19:26:35 +0000 Subject: [PATCH] First cut at value01.stg migation to ansible --- inventory/group_vars/value-stg | 27 + .../value01.stg.phx2.fedoraproject.org | 10 + playbooks/groups/value.yml | 50 ++ roles/fedmsg/irc/tasks/main.yml | 24 + .../irc/templates/fas-credentials.py.j2 | 6 + roles/fedmsg/irc/templates/ircbot.py.j2 | 145 ++++ roles/supybot/files/meetings_by_team.sh | 13 + roles/supybot/files/plugin.py | 797 ++++++++++++++++++ roles/supybot/tasks/main.yml | 25 + 9 files changed, 1097 insertions(+) create mode 100644 inventory/group_vars/value-stg create mode 100644 inventory/host_vars/value01.stg.phx2.fedoraproject.org create mode 100644 playbooks/groups/value.yml create mode 100644 roles/fedmsg/irc/tasks/main.yml create mode 100644 roles/fedmsg/irc/templates/fas-credentials.py.j2 create mode 100644 roles/fedmsg/irc/templates/ircbot.py.j2 create mode 100755 roles/supybot/files/meetings_by_team.sh create mode 100644 roles/supybot/files/plugin.py create mode 100644 roles/supybot/tasks/main.yml diff --git a/inventory/group_vars/value-stg b/inventory/group_vars/value-stg new file mode 100644 index 0000000000..5e5f4a5f32 --- /dev/null +++ b/inventory/group_vars/value-stg @@ -0,0 +1,27 @@ +--- +# Define resources for this group of hosts here. +lvm_size: 30000 +mem_size: 2048 +num_cpus: 2 + +# for systems that do not match the above - specify the same parameter in +# the host_vars/$hostname file + +tcp_ports: [ 80, 443, + # These 16 ports are used by fedmsg. One for each wsgi thread. + 3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007, + 3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015] + +# Neeed for rsync from log02 for logs. +custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.29 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.56 --dport 873 -j ACCEPT' ] + +fas_client_groups: sysadmin-noc,fi-apprentice + +# These are consumed by a task in roles/fedmsg/base/main.yml +fedmsg_certs: +- service: shell + owner: root + group: sysadmin +- service: supybot + owner: root + group: daemon diff --git a/inventory/host_vars/value01.stg.phx2.fedoraproject.org b/inventory/host_vars/value01.stg.phx2.fedoraproject.org new file mode 100644 index 0000000000..81e4235e8f --- /dev/null +++ b/inventory/host_vars/value01.stg.phx2.fedoraproject.org @@ -0,0 +1,10 @@ +--- +nm: 255.255.255.0 +gw: 10.5.126.254 +dns: 10.5.126.21 +ks_url: http://10.5.126.23/repo/rhel/ks/kvm-rhel-6 +ks_repo: http://10.5.126.23/repo/rhel/RHEL6-x86_64/ +volgroup: /dev/vg_guests +eth0_ip: 10.5.126.91 +vmhost: virthost12.phx2.fedoraproject.org +datacenter: phx2 diff --git a/playbooks/groups/value.yml b/playbooks/groups/value.yml new file mode 100644 index 0000000000..a2c340c583 --- /dev/null +++ b/playbooks/groups/value.yml @@ -0,0 +1,50 @@ +- name: make value add servers + hosts: value-stg + user: root + gather_facts: False + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + - include: "{{ tasks }}/accelerate_prep.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: value-stg + user: root + gather_facts: True + accelerate: "{{ accelerated }}" + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - denyhosts + - nagios_client + - fas_client + - collectd/base + - fedmsg/base + - fedmsg/irc + - supybot + + tasks: + - include: "{{ tasks }}/hosts.yml" + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/sudo.yml" + - include: "{{ tasks }}/openvpn_client.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/fedmsg/irc/tasks/main.yml b/roles/fedmsg/irc/tasks/main.yml new file mode 100644 index 0000000000..13962d2f3e --- /dev/null +++ b/roles/fedmsg/irc/tasks/main.yml @@ -0,0 +1,24 @@ + +- name: install fedmsg-irc + yum: pkg=fedmsg-irc state=installed + tags: + - packages + - fedmsg/irc + +- name: enable on boot and start fedmsg-irc + service: name=fedmsg-irc state=started enabled=true + tags: + - services + - fedmsg/irc + +- name: setup fedmsg-irc config file + template: src=ircbot.py.j2 dest=/etc/fedmsg.d/ircbot.py + tags: + - config + - fedmsg/irc + +- name: setup fas credentials config file + template: src=fas-credentials.py.j2 dest=/etc/fedmsg.d/fas-credentials.py + tags: + - config + - fedmsg/irc diff --git a/roles/fedmsg/irc/templates/fas-credentials.py.j2 b/roles/fedmsg/irc/templates/fas-credentials.py.j2 new file mode 100644 index 0000000000..ce141e6831 --- /dev/null +++ b/roles/fedmsg/irc/templates/fas-credentials.py.j2 @@ -0,0 +1,6 @@ +config = dict( + fas_credentials=dict( + username="fedoradummy", + password="<%= fedoraDummyUserPassword %>", + ), +) diff --git a/roles/fedmsg/irc/templates/ircbot.py.j2 b/roles/fedmsg/irc/templates/ircbot.py.j2 new file mode 100644 index 0000000000..28cd1692ab --- /dev/null +++ b/roles/fedmsg/irc/templates/ircbot.py.j2 @@ -0,0 +1,145 @@ +config = dict( + irc=[ + dict( + network='chat.freenode.net', + port=6667, + make_pretty=True, + make_terse=True, + + <% if environment == 'staging' %> + nickname='fedmsg-stg', + <% else %> + nickname='fedmsg-bot', + <% end %> + channel='fedora-fedmsg', + + # Ignore some of the koji spamminess + filters=dict( + topic=[ + 'buildsys.repo.init', + 'buildsys.repo.done', + 'buildsys.untag', + 'buildsys.tag', + ], + body=[], + ), + ), + + # Just for the Ask Fedora crew in #fedora-ask + dict( + network='chat.freenode.net', + port=6667, + make_pretty=True, + make_terse=True, + + <% if environment == 'staging' %> + nickname='fedmsg-ask-stg', + <% else %> + nickname='fedmsg-ask', + <% end %> + channel='fedora-ask', + # Only show AskFedora messages + filters=dict( + topic=['^((?!askbot).)*$'], + ), + ), + + # Show only pkgdb retirement msgs and compose msgs to the releng crew. + dict( + network='chat.freenode.net', + port=6667, + make_pretty=True, + make_terse=True, + + <% if environment == 'staging' %> + nickname='fedmsg-releng-s', + <% else %> + nickname='fedmsg-releng', + <% end %> + channel='fedora-releng', + filters=dict( + topic=[ + '^((?!(pkgdb\.package\.update\.status|compose|bodhi.updates.)).)*$', + ], + body=[ + "^((?!(u'status': u'Retired'|u'prev_status': u'Retired'|compose|bodhi\.updates\.|\/srv\/git\/releng)).)*$", + ], + ), + ), + + # The proyectofedora crew wants trac messages. + dict( + network='chat.freenode.net', + port=6667, + make_pretty=True, + make_terse=True, + + <% if environment == 'staging' %> + nickname='fedmsg-pfi-stg', + <% else %> + nickname='fedmsg-pfi', + <% end %> + channel='#proyecto-fedora', + # If the word proyecto appears in any message, forward it. + filters=dict( + body=['^((?!proyecto).)*$'], + ), + ), + + # Similarly for #fedora-latam. + dict( + network='chat.freenode.net', + port=6667, + make_pretty=True, + make_terse=True, + + <% if environment == 'staging' %> + nickname='fedmsg-latam-stg', + <% else %> + nickname='fedmsg-latam', + <% end %> + channel='#fedora-latam', + # If the word fedora-latam appears in any message, forward it. + filters=dict( + body=['^((?!fedora-latam).)*$'], + ), + ), + ], + + ### Possible colors are ### + # "white", + # "black", + # "blue", + # "green", + # "red", + # "brown", + # "purple", + # "orange", + # "yellow", + # "light green", + # "teal", + # "light cyan", + # "light blue", + # "pink", + # "grey", + # "light grey", + irc_color_lookup = { + "fas": "light blue", + "bodhi": "green", + "git": "red", + "fedoratagger": "brown", + "wiki": "purple", + "logger": "orange", + "pkgdb": "teal", + "buildsys": "yellow", + "fedoraplanet": "light green", + "trac": "pink", + "askbot": "light cyan", + "fedbadges": "brown", + "fedocal": "purple", + "copr": "red", + }, + + # This may be 'notice' or 'msg' + irc_method='msg', +) diff --git a/roles/supybot/files/meetings_by_team.sh b/roles/supybot/files/meetings_by_team.sh new file mode 100755 index 0000000000..b2486d992e --- /dev/null +++ b/roles/supybot/files/meetings_by_team.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +BASELOCATION=/srv/web/meetbot/teams +cd $BASELOCATION +cd .. + +for f in `find -type f -mtime -30 | grep -v "fedora-meeting\."` +do + teamname=$(basename $f | awk -F. '{ print $1 }' ) + mkdir -p $BASELOCATION/$teamname + ln -s $PWD/$f $BASELOCATION/$teamname/ 2> /dev/null +done + diff --git a/roles/supybot/files/plugin.py b/roles/supybot/files/plugin.py new file mode 100644 index 0000000000..97fa3a5416 --- /dev/null +++ b/roles/supybot/files/plugin.py @@ -0,0 +1,797 @@ +# -*- coding: utf-8 -*- +### +# Copyright (c) 2007, Mike McGrath +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions, and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions, and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the author of this software nor the name of +# contributors to this software may be used to endorse or promote products +# derived from this software without specific prior written consent. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +### + +import arrow +import sgmllib +import htmlentitydefs +import requests + +import supybot.utils as utils +import supybot.conf as conf +import time +from supybot.commands import * +import supybot.plugins as plugins +import supybot.ircutils as ircutils +import supybot.callbacks as callbacks + +from fedora.client import AppError +from fedora.client import AuthError +from fedora.client import ServerError +from fedora.client.fas2 import AccountSystem +from fedora.client.fas2 import FASError +from pkgdb2client import PkgDB + +from kitchen.text.converters import to_unicode + +import fedmsg.config +import fedmsg.meta + +import simplejson +import urllib +import commands +import urllib2 +import socket +import pytz +import datetime +import threading + +SPARKLINE_RESOLUTION = 50 + +datagrepper_url = 'https://apps.fedoraproject.org/datagrepper/raw' + + +def datagrepper_query(kwargs): + """ Return the count of msgs filtered by kwargs for a given time. + + The arguments for this are a little clumsy; this is imposed on us by + multiprocessing.Pool. + """ + start, end = kwargs.pop('start'), kwargs.pop('end') + params = { + 'start': time.mktime(start.timetuple()), + 'end': time.mktime(end.timetuple()), + } + params.update(kwargs) + + req = requests.get(datagrepper_url, params=params) + json_out = simplejson.loads(req.text) + result = int(json_out['total']) + return result + + +class WorkerThread(threading.Thread): + """ A simple worker thread for our threadpool. """ + + def __init__(self, fn, item, *args, **kwargs): + self.fn = fn + self.item = item + super(WorkerThread, self).__init__(*args, **kwargs) + + def run(self): + self.result = self.fn(self.item) + + +class ThreadPool(object): + """ Our very own threadpool implementation. + + We make our own thing because multiprocessing is too heavy. + """ + + def map(self, fn, items): + threads = [] + + for item in items: + threads.append(WorkerThread(fn=fn, item=item)) + + for thread in threads: + thread.start() + + for thread in threads: + thread.join() + + return [thread.result for thread in threads] + + +class Title(sgmllib.SGMLParser): + entitydefs = htmlentitydefs.entitydefs.copy() + entitydefs['nbsp'] = ' ' + + def __init__(self): + self.inTitle = False + self.title = '' + sgmllib.SGMLParser.__init__(self) + + def start_title(self, attrs): + self.inTitle = True + + def end_title(self): + self.inTitle = False + + def unknown_entityref(self, name): + if self.inTitle: + self.title += ' ' + + def unknown_charref(self, name): + if self.inTitle: + self.title += ' ' + + def handle_data(self, data): + if self.inTitle: + self.title += data + + +class Fedora(callbacks.Plugin): + """Use this plugin to retrieve Fedora-related information.""" + threaded = True + + def __init__(self, irc): + super(Fedora, self).__init__(irc) + + # caches, automatically downloaded on __init__, manually refreshed on + # .refresh + self.userlist = None + self.bugzacl = None + + # To get the information, we need a username and password to FAS. + # DO NOT COMMIT YOUR USERNAME AND PASSWORD TO THE PUBLIC REPOSITORY! + self.fasurl = self.registryValue('fas.url') + self.username = self.registryValue('fas.username') + self.password = self.registryValue('fas.password') + + self.fasclient = AccountSystem(self.fasurl, username=self.username, + password=self.password) + self.pkgdb = PkgDB() + # URLs + #self.url = {} + + # fetch necessary caches + self._refresh() + + # Pull in /etc/fedmsg.d/ so we can build the fedmsg.meta processors. + fm_config = fedmsg.config.load_config() + fedmsg.meta.make_processors(**fm_config) + + def _refresh(self): + timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(None) + self.log.info("Downloading user data") + request = self.fasclient.send_request('/user/list', + req_params={'search': '*'}, + auth=True, + timeout=240) + users = request['people'] + request['unapproved_people'] + del request + self.log.info("Caching necessary user data") + self.users = {} + self.faslist = {} + for user in users: + name = user['username'] + self.users[name] = {} + self.users[name]['id'] = user['id'] + key = ' '.join([user['username'], user['email'] or '', + user['human_name'] or '', user['ircnick'] or '']) + key = key.lower() + value = "%s '%s' <%s>" % (user['username'], user['human_name'] or + '', user['email'] or '') + self.faslist[key] = value + self.log.info("Downloading package owners cache") + data = requests.get( + 'https://admin.fedoraproject.org/pkgdb/api/bugzilla?format=json', + verify=True).json() + self.bugzacl = data['bugzillaAcls'] + socket.setdefaulttimeout(timeout) + + def refresh(self, irc, msg, args): + """takes no arguments + + Refresh the necessary caches.""" + self._refresh() + irc.replySuccess() + refresh = wrap(refresh) + + def _load_json(self, url): + timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(45) + json = simplejson.loads(utils.web.getUrl(url)) + socket.setdefaulttimeout(timeout) + return json + + def whoowns(self, irc, msg, args, package): + """ + + Retrieve the owner of a given package + """ + try: + mainowner = self.bugzacl['Fedora'][package]['owner'] + except KeyError: + irc.reply("No such package exists.") + return + others = [] + for key in self.bugzacl: + if key == 'Fedora': + continue + try: + owner = self.bugzacl[key][package]['owner'] + if owner == mainowner: + continue + except KeyError: + continue + others.append("%s in %s" % (owner, key)) + if others == []: + irc.reply(mainowner) + else: + irc.reply("%s (%s)" % (mainowner, ', '.join(others))) + whoowns = wrap(whoowns, ['text']) + + def branches(self, irc, msg, args, package): + """ + + Return the branches a package is in.""" + try: + pkginfo = self.pkgdb.get_package(package) + except AppError: + irc.reply("No such package exists.") + return + branch_list = [] + for listing in pkginfo['packages']: + branch_list.append(listing['collection']['branchname']) + branch_list.sort() + irc.reply(' '.join(branch_list)) + return + branches = wrap(branches, ['text']) + + def what(self, irc, msg, args, package): + """ + + Returns a description of a given package. + """ + try: + summary = self.bugzacl['Fedora'][package]['summary'] + irc.reply("%s: %s" % (package, summary)) + except KeyError: + irc.reply("No such package exists.") + return + what = wrap(what, ['text']) + + def fas(self, irc, msg, args, find_name): + """ + + Search the Fedora Account System usernames, full names, and email + addresses for a match.""" + find_name = to_unicode(find_name) + matches = [] + for entry in self.faslist.keys(): + if entry.find(find_name.lower()) != -1: + matches.append(entry) + if len(matches) == 0: + irc.reply("'%s' Not Found!" % find_name) + else: + output = [] + for match in matches: + output.append(self.faslist[match]) + irc.reply(' - '.join(output).encode('utf-8')) + fas = wrap(fas, ['text']) + + def hellomynameis(self, irc, msg, args, name): + """ + + Return brief information about a Fedora Account System username. Useful + for things like meeting roll call and calling attention to yourself.""" + try: + person = self.fasclient.person_by_username(name) + except: + irc.reply('Something blew up, please try again') + return + if not person: + irc.reply('Sorry, but you don\'t exist') + return + irc.reply(('%(username)s \'%(human_name)s\' <%(email)s>' % + person).encode('utf-8')) + hellomynameis = wrap(hellomynameis, ['text']) + + def himynameis(self, irc, msg, args, name): + """ + + Will the real Slim Shady please stand up?""" + try: + person = self.fasclient.person_by_username(name) + except: + irc.reply('Something blew up, please try again') + return + if not person: + irc.reply('Sorry, but you don\'t exist') + return + irc.reply(('%(username)s \'Slim Shady\' <%(email)s>' % + person).encode('utf-8')) + himynameis = wrap(himynameis, ['text']) + + def localtime(self, irc, msg, args, name): + """ + + Returns the current time of the user. + The timezone is queried from FAS.""" + try: + person = self.fasclient.person_by_username(name) + except: + irc.reply('Error getting info user user: "%s"' % name) + return + if not person: + irc.reply('User "%s" doesn\'t exist' % name) + return + timezone_name = person['timezone'] + if timezone_name is None: + irc.reply('User "%s" doesn\'t share his timezone' % name) + return + try: + time = datetime.datetime.now(pytz.timezone(timezone_name)) + except: + irc.reply('The timezone of "%s" was unknown: "%s"' % (name, + timezone)) + return + irc.reply('The current local time of "%s" is: "%s" (timezone: %s)' % + (name, time.strftime('%H:%M'), timezone_name)) + localtime = wrap(localtime, ['text']) + + def fasinfo(self, irc, msg, args, name): + """ + + Return information on a Fedora Account System username.""" + try: + person = self.fasclient.person_by_username(name) + except: + irc.reply('Error getting info for user: "%s"' % name) + return + if not person: + irc.reply('User "%s" doesn\'t exist' % name) + return + person['creation'] = person['creation'].split(' ')[0] + string = ("User: %(username)s, Name: %(human_name)s" + ", email: %(email)s, Creation: %(creation)s" + ", IRC Nick: %(ircnick)s, Timezone: %(timezone)s" + ", Locale: %(locale)s" + ", GPG key ID: %(gpg_keyid)s, Status: %(status)s") % person + irc.reply(string.encode('utf-8')) + + # List of unapproved groups is easy + unapproved = '' + for group in person['unapproved_memberships']: + unapproved = unapproved + "%s " % group['name'] + if unapproved != '': + irc.reply('Unapproved Groups: %s' % unapproved) + + # List of approved groups requires a separate query to extract roles + constraints = {'username': name, 'group': '%', + 'role_status': 'approved'} + columns = ['username', 'group', 'role_type'] + roles = [] + try: + roles = self.fasclient.people_query(constraints=constraints, + columns=columns) + except: + irc.reply('Error getting group memberships.') + return + + approved = '' + for role in roles: + if role['role_type'] == 'sponsor': + approved += '+' + role['group'] + ' ' + elif role['role_type'] == 'administrator': + approved += '@' + role['group'] + ' ' + else: + approved += role['group'] + ' ' + if approved == '': + approved = "None" + + irc.reply('Approved Groups: %s' % approved) + fasinfo = wrap(fasinfo, ['text']) + + def group(self, irc, msg, args, name): + """ + + Return information about a Fedora Account System group.""" + try: + group = self.fasclient.group_by_name(name) + irc.reply('%s: %s' % + (name, group['display_name'])) + except AppError: + irc.reply('There is no group "%s".' % name) + group = wrap(group, ['text']) + + def admins(self, irc, msg, args, name): + """ + + Return the administrators list for the selected group""" + + try: + group = self.fasclient.group_members(name) + sponsors = '' + for person in group: + if person['role_type'] == 'administrator': + sponsors += person['username'] + ' ' + irc.reply('Administrators for %s: %s' % (name, sponsors)) + except AppError: + irc.reply('There is no group %s.' % name) + + admins = wrap(admins, ['text']) + + def sponsors(self, irc, msg, args, name): + """ + + Return the sponsors list for the selected group""" + + try: + group = self.fasclient.group_members(name) + sponsors = '' + for person in group: + if person['role_type'] == 'sponsor': + sponsors += person['username'] + ' ' + elif person['role_type'] == 'administrator': + sponsors += '@' + person['username'] + ' ' + irc.reply('Sponsors for %s: %s' % (name, sponsors)) + except AppError: + irc.reply('There is no group %s.' % name) + + sponsors = wrap(sponsors, ['text']) + + def members(self, irc, msg, args, name): + """ + + Return a list of members of the specified group""" + try: + group = self.fasclient.group_members(name) + members = '' + for person in group: + if person['role_type'] == 'administrator': + members += '@' + person['username'] + ' ' + elif person['role_type'] == 'sponsor': + members += '+' + person['username'] + ' ' + else: + members += person['username'] + ' ' + irc.reply('Members of %s: %s' % (name, members)) + except AppError: + irc.reply('There is no group %s.' % name) + + members = wrap(members, ['text']) + + def showticket(self, irc, msg, args, baseurl, number): + """ + + Return the name and URL of a trac ticket or bugzilla bug. + """ + url = format(baseurl, str(number)) + size = conf.supybot.protocols.http.peekSize() + text = utils.web.getUrl(url, size=size) + parser = Title() + try: + parser.feed(text) + except sgmllib.SGMLParseError: + irc.reply(format('Encountered a problem parsing %u', url)) + if parser.title: + irc.reply(utils.web.htmlToText(parser.title.strip()) + ' - ' + url) + else: + irc.reply(format('That URL appears to have no HTML title ' + + 'within the first %i bytes.', size)) + showticket = wrap(showticket, ['httpUrl', 'int']) + + def swedish(self, irc, msg, args): + """takes no arguments + + Humor mmcgrath.""" + + # Import this here to avoid a circular import problem. + from __init__ import __version__ + + irc.reply(str('kwack kwack')) + irc.reply(str('bork bork bork')) + irc.reply(str('(supybot-fedora version %s)' % __version__)) + swedish = wrap(swedish) + + def wikilink(self, irc, msg, args, name): + """ + + Return MediaWiki link syntax for a FAS user's page on the wiki.""" + try: + person = self.fasclient.person_by_username(name) + except: + irc.reply('Error getting info for user: "%s"' % name) + return + if not person: + irc.reply('User "%s" doesn\'t exist' % name) + return + string = "[[User:%s|%s]]" % (person["username"], + person["human_name"] or '') + irc.reply(string.encode('utf-8')) + wikilink = wrap(wikilink, ['text']) + + def mirroradmins(self, irc, msg, args, hostname): + """ + + Return MirrorManager list of FAS usernames which administer . + must be the FQDN of the host.""" + url = ("https://admin.fedoraproject.org/mirrormanager/mirroradmins?" + "tg_format=json&host=" + hostname) + result = self._load_json(url)['values'] + if len(result) == 0: + irc.reply('Hostname "%s" not found' % hostname) + return + string = 'Mirror Admins of %s: ' % hostname + string += ' '.join(result) + irc.reply(string.encode('utf-8')) + mirroradmins = wrap(mirroradmins, ['text']) + + def nextmeeting(self, irc, msg, args, channel): + """ + + Return the next meeting scheduled for a particular channel. + """ + + channel = channel.strip('#').split('@')[0] + meetings = list(self._future_meetings(channel)) + if not meetings: + response = "There are no meetings scheduled for #%s." % channel + irc.reply(response.encode('utf-8')) + return + + date, meeting = meetings[0] + response = "The next meeting in #%s is %s (starting %s)" % ( + channel, + meeting['meeting_name'], + arrow.get(date).humanize(), + ) + irc.reply(response.encode('utf-8')) + base = "https://apps.fedoraproject.org/calendar/location/" + url = base + urllib.quote("%s@irc.freenode.net/" % channel) + irc.reply("- " + url.encode('utf-8')) + nextmeeting = wrap(nextmeeting, ['text']) + + @staticmethod + def _future_meetings(channel): + response = requests.get( + 'https://apps.fedoraproject.org/calendar/api/meetings', + params=dict( + location='%s@irc.freenode.net' % channel, + ) + ) + + data = response.json() + now = datetime.datetime.utcnow() + + for meeting in data['meetings']: + string = meeting['meeting_date'] + " " + meeting['meeting_time_start'] + dt = datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S") + + if now < dt: + yield dt, meeting + + def badges(self, irc, msg, args, name): + """ + + Return badges statistics about a user. + """ + url = "https://badges.fedoraproject.org/user/" + name + d = requests.get(url + "/json").json() + + if 'error' in d: + response = d['error'] + else: + template = "{name} has unlocked {n} Fedora Badges: {url}" + n = len(d['assertions']) + response = template.format(name=name, url=url, n=n) + + irc.reply(response.encode('utf-8')) + badges = wrap(badges, ['text']) + + def quote(self, irc, msg, args, arguments): + """ [daily, weekly, monthly, quarterly] + + Return some datagrepper statistics on fedmsg categories. + """ + + # First, some argument parsing. Supybot should be able to do this for + # us, but I couldn't figure it out. The supybot.plugins.additional + # object is the thing to use... except its weird. + tokens = arguments.split(None, 1) + if len(tokens) == 1: + symbol, frame = tokens[0], 'daily' + else: + symbol, frame = tokens + + # Second, build a lookup table for symbols. By default, we'll use the + # fedmsg category names, take their first 3 characters and uppercase + # them. That will take things like "wiki" and turn them into "WIK" and + # "bodhi" and turn them into "BOD". This handles a lot for us. We'll + # then override those that don't make sense manually here. For + # instance "fedoratagger" by default would be "FED", but that's no + # good. We want "TAG". + # Why all this trouble? Well, as new things get added to the fedmsg + # bus, we don't want to have keep coming back here and modifying this + # code. Hopefully this dance will at least partially future-proof us. + symbols = dict([ + (processor.__name__.lower(), processor.__name__[:3].upper()) + for processor in fedmsg.meta.processors + ]) + symbols.update({ + 'fedoratagger': 'TAG', + 'fedbadges': 'BDG', + 'buildsys': 'KOJ', + 'pkgdb': 'PKG', + 'meetbot': 'MTB', + 'planet': 'PLN', + 'trac': 'TRC', + 'mailman': 'MM3', + }) + + # Now invert the dict so we can lookup the argued symbol. + # Yes, this is vulnerable to collisions. + symbols = dict([(sym, name) for name, sym in symbols.items()]) + + # These aren't user-facing topics, so drop 'em. + del symbols['LOG'] + del symbols['UNH'] + del symbols['ANN'] # And this one is unused... + + key_fmt = lambda d: ', '.join(sorted(d.keys())) + + if not symbol in symbols: + response = "No such symbol %r. Try one of %s" + irc.reply((response % (symbol, key_fmt(symbols))).encode('utf-8')) + return + + # Now, build another lookup of our various timeframes. + frames = dict( + daily=datetime.timedelta(days=1), + weekly=datetime.timedelta(days=7), + monthly=datetime.timedelta(days=30), + quarterly=datetime.timedelta(days=91), + ) + + if not frame in frames: + response = "No such timeframe %r. Try one of %s" + irc.reply((response % (frame, key_fmt(frames))).encode('utf-8')) + return + + category = [symbols[symbol]] + + t2 = datetime.datetime.now() + t1 = t2 - frames[frame] + t0 = t1 - frames[frame] + + # Count the number of messages between t0 and t1, and between t1 and t2 + query1 = dict(start=t0, end=t1, category=category) + query2 = dict(start=t1, end=t2, category=category) + + # Do this async for superfast datagrepper queries. + tpool = ThreadPool() + batched_values = tpool.map(datagrepper_query, [ + dict(start=x, end=y, category=category) + for x, y in Utils.daterange(t1, t2, SPARKLINE_RESOLUTION) + ] + [query1, query2]) + + count2 = batched_values.pop() + count1 = batched_values.pop() + + # Just rename the results. We'll use the rest for the sparkline. + sparkline_values = batched_values + + yester_phrases = dict( + daily="yesterday", + weekly="the week preceding this one", + monthly="the month preceding this one", + quarterly="the 3 months preceding these past three months", + ) + phrases = dict( + daily="24 hours", + weekly="week", + monthly="month", + quarterly="3 months", + ) + + if count1 and count2: + percent = ((float(count2) / count1) - 1) * 100 + elif not count1 and count2: + # If the older of the two time periods had zero messages, but there + # are some in the more current period.. well, that's an infinite + # percent increase. + percent = float('inf') + elif not count1 and not count2: + # If counts are zero for both periods, then the change is 0%. + percent = 0 + else: + # Else, if there were some messages in the old time period, but + # none in the current... then that's a 100% drop off. + percent = -100 + + sign = lambda value: value >= 0 and '+' or '-' + + template = u"{sym}, {name} {sign}{percent:.2f}% over {phrase}" + response = template.format( + sym=symbol, + name=symbols[symbol], + sign=sign(percent), + percent=abs(percent), + phrase=yester_phrases[frame], + ) + irc.reply(response.encode('utf-8')) + + # Now, make a graph out of it. + sparkline = Utils.sparkline(sparkline_values) + + template = u" {sparkline} ⤆ over {phrase}" + response = template.format( + sym=symbol, + sparkline=sparkline, + phrase=phrases[frame] + ) + irc.reply(response.encode('utf-8')) + + to_utc = lambda t: time.gmtime(time.mktime(t.timetuple())) + # And a final line for "x-axis tics" + t1_fmt = time.strftime("%H:%M UTC %m/%d", to_utc(t1)) + t2_fmt = time.strftime("%H:%M UTC %m/%d", to_utc(t2)) + padding = u" " * (SPARKLINE_RESOLUTION - len(t1_fmt) - 3) + template = u" ↑ {t1}{padding}↑ {t2}" + response = template.format(t1=t1_fmt, t2=t2_fmt, padding=padding) + irc.reply(response.encode('utf-8')) + quote = wrap(quote, ['text']) + + +class Utils(object): + """ Some handy utils for datagrepper visualization. """ + + @classmethod + def sparkline(cls, values): + bar = u'▁▂▃▄▅▆▇█' + barcount = len(bar) - 1 + values = map(float, values) + mn, mx = min(values), max(values) + extent = mx - mn + + if extent == 0: + indices = [0 for n in values] + else: + indices = [int((n - mn) / extent * barcount) for n in values] + + unicode_sparkline = u''.join([bar[i] for i in indices]) + return unicode_sparkline + + @classmethod + def daterange(cls, start, stop, steps): + """ A generator for stepping through time. """ + delta = (stop - start) / steps + current = start + while current + delta <= stop: + yield current, current + delta + current += delta + + +Class = Fedora + + +# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/roles/supybot/tasks/main.yml b/roles/supybot/tasks/main.yml new file mode 100644 index 0000000000..b76b65f7d0 --- /dev/null +++ b/roles/supybot/tasks/main.yml @@ -0,0 +1,25 @@ +- name: install supybot package + yum: pkg={{ item }} state=installed + with_items: + - supybot-gribble + - supybot-fedora + - supybot-koji + - supybot-notify + - supybot-pinglists + +- name: creating zodbot log dir + file: path={{ item }} state=directory owner=daemon + with_items: + - /var/lib/zodbot + +- name: setup meetings_by_team script + copy: src=meetings_by_team dest=/usr/local/bin/meetings_by_team.sh mode=755 + +- name: teams cron job + cron: name=meetings-by-team hour="23" minute="0" user=daemon job="/usr/local/bin/meetings_by_team.sh" + +- name: hotfix - packagedb-cli which is a new dep but is not there in the rpm + yum: pkg=packagedb-cli state=installed + +- name: hotfix - supybot plugin + copy: src=plugin.py dest=/usr/lib/python2.6/site-packages/supybot/plugins/Fedora/plugin.py mode=755 owner=root