First stab at a fedmsg base and hub tasks

This commit is contained in:
Kevin Fenzi 2013-06-07 20:01:23 +00:00
parent 24f965de22
commit 08f2e96bd9
9 changed files with 677 additions and 0 deletions

47
files/fedmsg/base.py.j2 Normal file
View file

@ -0,0 +1,47 @@
config = dict(
# Set this to dev if you're hacking on fedmsg or an app locally.
# Set to stg or prod if running in the Fedora Infrastructure.
{% if environment == 'staging' %}
environment="stg",
{% else %}
environment="prod",
{% endif %}
# This used to be set to 1 for safety, but it turns out it was
# excessive. It is the number of seconds that fedmsg should sleep
# after it has initialized, but before it begins to try and send any
# messages. If set to a non-zero value, this will slow down one-off
# fedmsg scripts like the git post-receive hook and pkgdb2branch.
# If we are experiencing message-loss problems, one of the first things
# to try should be to turn this number up to a non-zero value. '1' should
# be more than sufficient.
post_init_sleep=0.4,
# This is the number of milliseconds to wait before timing out on
# connections.. notably to the fedmsg-relay in the event that it has
# crashed.
zmq_linger=2000,
# Default is 0
high_water_mark=0,
io_threads=1,
# We almost always want the fedmsg-hub to be sending messages with zmq as
# opposed to amqp or stomp. The only exception will be the bugzilla
# amqp<->zmq bridge service.
zmq_enabled=True,
# When subscribing to messages, we want to allow splats ('*') so we tell the
# hub to not be strict when comparing messages topics to subscription
# topics.
zmq_strict=False,
# See the following
# - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html
# - http://api.zeromq.org/3-2:zmq-setsockopt
zmq_tcp_keepalive=1,
zmq_tcp_keepalive_cnt=3,
zmq_tcp_keepalive_idle=60,
zmq_tcp_keepalive_intvl=5,
)

View file

@ -0,0 +1,126 @@
{% if environment == 'staging' %}
suffix = 'stg.phx2.fedoraproject.org'
non_phx_suffix = 'stg.fedoraproject.org'
{% else %}
suffix = 'phx2.fedoraproject.org'
non_phx_suffix = 'fedoraproject.org'
vpn_suffix = 'vpn.fedoraproject.org'
{% endif %}
config = dict(
# This is a dict of possible addresses from which fedmsg can send
# messages. fedmsg.init(...) requires that a 'name' argument be passed
# to it which corresponds with one of the keys in this dict.
endpoints = {
# For message producers, fedmsg will try to guess the
# name of it's calling module to determine which endpoint definition
# to use. This can be overridden by explicitly providing the name in
# the initial call to fedmsg.init(...).
"bodhi.app01": [
"tcp://app01.%s:300%i" % (suffix, i)
for i in range(8)
],
"bodhi.app02": [
"tcp://app02.%s:300%i" % (suffix, i)
for i in range(8)
],
"bodhi.releng01": [
"tcp://releng01.%s:3000" % suffix,
"tcp://releng01.%s:3001" % suffix,
],
{% if environment != 'staging' %}
"bodhi.app03": [
"tcp://app03.%s:300%i" % (suffix, i)
for i in range(8)
],
"bodhi.app04": [
"tcp://app04.%s:300%i" % (suffix, i)
for i in range(8)
],
"bodhi.app05": [
"tcp://app05.%s:300%i" % (non_phx_suffix, i)
for i in range(8)
],
"bodhi.app06": [
"tcp://app06.%s:300%i" % (non_phx_suffix, i)
for i in range(8)
],
"bodhi.app07": [
"tcp://app07.%s:300%i" % (suffix, i)
for i in range(8)
],
"bodhi.app08": [
"tcp://app08.%s:300%i" % (non_phx_suffix, i)
for i in range(8)
],
"bodhi.releng04": [
"tcp://releng04.%s:3000" % suffix,
"tcp://releng04.%s:3001" % suffix,
],
"bodhi.relepel01": [
"tcp://relepel01.%s:3000" % suffix,
"tcp://relepel01.%s:3001" % suffix,
],
{% endif %}
# FAS is a little out of the ordinary. It has 32 endpoints instead of
# the usual 8 since there are so many mod_wsgi processes for it.
"fas.fas01": [
"tcp://fas01.%s:30%02i" % (suffix, i)
for i in range(32)
],
{% if environment != 'staging' %}
"fas.fas02": [
"tcp://fas02.%s:30%02i" % (suffix, i)
for i in range(32)
],
"fas.fas03": [
"tcp://fas03.%s:30%02i" % (suffix, i)
for i in range(32)
],
{% endif %}
# Well, fedoratagger needs 32 endpoints too, just like FAS.
"fedoratagger.packages01": [
"tcp://packages01.%s:30%02i" % (suffix, i)
for i in range(32)
],
{% if environment != 'staging' %}
"fedoratagger.packages02": [
"tcp://packages02.%s:30%02i" % (suffix, i)
for i in range(32)
],
{% endif %}
"busmon_consumers.busgateway01": [
"tcp://busgateway01.%s:3000" % suffix,
],
{% if environment != 'staging' %}
"supybot.value03": [
"tcp://value03.%s:3000" % suffix,
],
{% endif %}
# Askbot runs as 6 processes with 1 thread each.
"askbot.ask01": [
"tcp://ask01.%s:30%02i" % (suffix, i)
for i in range(6)
],
# Askbot runs as 6 processes with 1 thread each.
"askbot.ask02": [
"tcp://ask02.%s:30%02i" % (suffix, i)
for i in range(6)
],
{% if environment != 'staging' %}
# fedorahosted trac runs as 4 processes with 4 threads each.
"trac.hosted03": [
"tcp://hosted03.%s:30%02i" % (vpn_suffix, i)
for i in range(16)
],
"trac.hosted04": [
"tcp://hosted04.%s:30%02i" % (vpn_suffix, i)
for i in range(16)
],
{% endif %}
# koji is not listed here since it publishes to the fedmsg-relay
},
)

View file

@ -0,0 +1,32 @@
# Setup fedmsg logging.
# See the following for constraints on this format http://bit.ly/Xn1WDn
config = dict(
logging=dict(
version=1,
formatters=dict(
bare={
"format": "%(message)s",
},
),
handlers=dict(
console={
"class": "logging.StreamHandler",
"formatter": "bare",
"level": "DEBUG",
"stream": "ext://sys.stdout",
}
),
loggers=dict(
fedmsg={
"level": "DEBUG",
"propagate": False,
"handlers": ["console"],
},
moksha={
"level": "DEBUG",
"propagate": False,
"handlers": ["console"],
},
),
),
)

46
files/fedmsg/pkgdb.py.j2 Normal file
View file

@ -0,0 +1,46 @@
{% if environment == 'staging' %}
suffix = 'stg.phx2.fedoraproject.org'
non_phx_suffix = 'stg.fedoraproject.org'
{% else %}
suffix = 'phx2.fedoraproject.org'
non_phx_suffix = 'fedoraproject.org'
{% end %}
config = dict(
endpoints={
"pkgdb.app01": [
"tcp://app01.%s:301%i" % (suffix, i)
for i in range(6)
],
"pkgdb.app02": [
"tcp://app02.%s:301%i" % (suffix, i)
for i in range(6)
],
{% if environment != 'staging' %}
"pkgdb.app03": [
"tcp://app03.%s:301%i" % (suffix, i)
for i in range(6)
],
"pkgdb.app04": [
"tcp://app04.%s:301%i" % (suffix, i)
for i in range(6)
],
"pkgdb.app05": [
"tcp://app05.%s:301%i" % (non_phx_suffix, i)
for i in range(6)
],
"pkgdb.app06": [
"tcp://app06.%s:301%i" % (non_phx_suffix, i)
for i in range(6)
],
"pkgdb.app07": [
"tcp://app07.%s:301%i" % (suffix, i)
for i in range(6)
],
"pkgdb.app08": [
"tcp://app08.%s:301%i" % (non_phx_suffix, i)
for i in range(6)
],
{% endif %}
},
)

39
files/fedmsg/relay.py.j2 Normal file
View file

@ -0,0 +1,39 @@
{% if environment == 'staging' %}
suffix = 'stg.phx2.fedoraproject.org'
non_phx_suffix = 'stg.fedoraproject.org'
{% else %}
suffix = 'phx2.fedoraproject.org'
non_phx_suffix = 'fedoraproject.org'
{% endif %}
# This is just an extension of fedmsg.d/endpoints.py. This dict
# will get merged in with the other.
config = dict(
endpoints={
# This is the output side of the relay to which all other
# services can listen.
"relay_outbound": [
# Messages from inside phx2 and the vpn emerge here
"tcp://app01.%s:3999" % suffix,
# Messages from coprs and secondary arch composes emerge here
"tcp://busgateway01.%s:3999" % suffix,
],
},
# This is the address of an active->passive relay. It is used for the
# fedmsg-logger command which requires another service with a stable
# listening address for it to send messages to.
# It is also used by the git-hook, for the same reason.
# It is also used by the mediawiki php plugin which, due to the oddities of
# php, can't maintain a single passive-bind endpoint of it's own.
relay_inbound=[
# Scripts inside phx2 connect here
"tcp://app01.%s:3998" % suffix,
# Scripts from the vpn (people03) connect here
"tcp://app01.vpn.fedoraproject.org:3998",
# Scripts from outside connect here (coprs, secondary arch composes)
"tcp://busgateway01.%s:9941" % suffix,
],
)

320
files/fedmsg/ssl.py.j2 Normal file
View file

@ -0,0 +1,320 @@
{% if environment = 'staging' %}
suffix = "stg.phx2.fedoraproject.org"
app_hosts = [
"app01.stg.phx2.fedoraproject.org",
"app02.stg.phx2.fedoraproject.org",
]
topic_prefix = "org.fedoraproject.stg."
{% else %}
suffix = "phx2.fedoraproject.org"
app_hosts = [
"app01.phx2.fedoraproject.org",
"app02.phx2.fedoraproject.org",
"app03.phx2.fedoraproject.org",
"app04.phx2.fedoraproject.org",
"app05.fedoraproject.org",
"app06.fedoraproject.org",
"app07.phx2.fedoraproject.org",
"app08.fedoraproject.org",
]
topic_prefix = "org.fedoraproject.prod."
{% endif %}
vpn_suffix = "vpn.fedoraproject.org"
config = dict(
sign_messages=True,
validate_signatures=True,
ssldir="/etc/pki/fedmsg",
crl_location="https://fedoraproject.org/fedmsg/crl.pem",
crl_cache="/var/run/fedmsg/crl.pem",
crl_cache_expiry=86400, # Daily
certnames=dict(
[
("shell.app0%i" % i, "shell-%s" % app_hosts[i-1])
for i in range(1, len(app_hosts) + 1)
] + [
("bodhi.app0%i" % i, "bodhi-%s" % app_hosts[i-1])
for i in range(1, len(app_hosts) + 1)
] + [
("pkgdb.app0%i" % i, "pkgdb-%s" % app_hosts[i-1])
for i in range(1, len(app_hosts) + 1)
] + [
("mediawiki.app0%i" % i, "mediawiki-%s" % app_hosts[i-1])
for i in range(1, len(app_hosts) + 1)
] + [
("shell.fas0%i" % i, "shell-fas0%i.%s" % (i, suffix))
for i in range(1, 4)
] + [
("fas.fas0%i" % i, "fas-fas0%i.%s" % (i, suffix))
for i in range(1, 4)
] + [
("shell.packages0%i" % i, "shell-packages0%i.%s" % (i, suffix))
for i in range(1, 3)
] + [
("fedoratagger.packages0%i" % i, "fedoratagger-packages0%i.%s" % (i, suffix))
for i in range(1, 3)
] + [
("shell.pkgs0%i" % i, "shell-pkgs0%i.%s" % (i, suffix))
for i in range(1, 2)
] + [
("scm.pkgs0%i" % i, "scm-pkgs0%i.%s" % (i, suffix))
for i in range(1, 2)
] + [
("lookaside.pkgs0%i" % i, "lookaside-pkgs0%i.%s" % (i, suffix))
for i in range(1, 2)
] + [
("shell.relepel01", "shell-relepel01.%s" % suffix),
("shell.releng04", "shell-releng04.%s" % suffix),
("shell.releng01", "shell-releng01.%s" % suffix),
("shell.releng03", "shell-releng03.%s" % suffix),
("bodhi.relepel01", "bodhi-relepel01.%s" % suffix),
("bodhi.releng04", "bodhi-releng04.%s" % suffix),
("bodhi.releng01", "bodhi-releng01.%s" % suffix),
("bodhi.releng03", "bodhi-releng03.%s" % suffix),
] + [
("busmon_consumers.busgateway01", "busmon-busgateway01.%s" % suffix),
("shell.busgateway01", "shell-busgateway01.%s" % suffix),
] + [
("shell.value01", "shell-value01.%s" % suffix),
("shell.value03", "shell-value03.%s" % suffix),
("supybot.value03", "supybot-value03.%s" % suffix),
] + [
("koji.koji04", "koji-koji04.%s" % suffix),
("koji.koji01", "koji-koji01.%s" % suffix),
("koji.koji03", "koji-koji03.%s" % suffix),
("shell.koji04", "shell-koji04.%s" % suffix),
("shell.koji01", "shell-koji01.%s" % suffix),
("shell.koji03", "shell-koji03.%s" % suffix),
] + [
("nagios.noc01", "nagios-noc01.%s" % suffix),
("shell.noc01", "shell-noc01.%s" % suffix),
] + [
("git.hosted03", "git-hosted03.%s" % vpn_suffix),
("git.hosted04", "git-hosted04.%s" % vpn_suffix),
("trac.hosted03", "trac-hosted03.%s" % vpn_suffix),
("trac.hosted04", "trac-hosted04.%s" % vpn_suffix),
("shell.hosted03", "shell-hosted03.%s" % vpn_suffix),
("shell.hosted04", "shell-hosted04.%s" % vpn_suffix),
] + [
("shell.lockbox01", "shell-lockbox01.%s" % suffix),
("announce.lockbox01", "announce-lockbox01.%s" % suffix),
] + [
# These first two entries are here to placate a bug in
# python-askbot-fedmsg-0.0.4. They can be removed once
# python-askbot-fedmsg-0.0.5 hits town.
("askbot.ask01.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix),
("askbot.ask01.stg.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix),
("askbot.ask01", "askbot-ask01.%s" % suffix),
("shell.ask01", "shell-ask01.%s" % suffix),
("askbot.ask02", "askbot-ask02.%s" % suffix),
("shell.ask02", "shell-ask02.%s" % suffix),
]),
routing_policy={
# The gist here is that only messages signed by the
# bodhi-app0{1,2,3,4,5,6,7,8} certificates may bear the
# "org.fedoraproject.prod.bodhi.update.request.stable" topic, or else
# they fail validation and are either dropped or marked as invalid
# (depending on the consumer's wishes).
#
# There is another option that we do not set. If `routing_nitpicky` is
# set to True, then a given message's topic *must* appear in this list
# in order for it to pass validation. For instance, we have
# routing_nitpicky set to False by default and no
# "org.fedoraproject.prod.logger.log" topics appear in this policy,
# therefore, any message bearing that topic and *any* certificate signed
# by our CA may pass validation.
#
topic_prefix + "bodhi.update.request.stable": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.update.request.testing": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.update.request.unpush": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.update.comment": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.buildroot_override.tag": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.buildroot_override.untag": [
"bodhi-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "bodhi.mashtask.mashing": [
"bodhi-releng04.%s" % suffix,
"bodhi-relepel01.%s" % suffix,
],
topic_prefix + "bodhi.mashtask.complete": [
"bodhi-releng04.%s" % suffix,
"bodhi-relepel01.%s" % suffix,
],
# Compose (rel-eng) messages (use the bodhi certs)
topic_prefix + "compose.rawhide.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.rawhide.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.rawhide.mash.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.rawhide.mash.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.rawhide.rsync.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.rawhide.rsync.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.pungify.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.pungify.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.mash.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.mash.complete": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.rsync.start": [
"bodhi-releng03.%s" % suffix,
],
topic_prefix + "compose.branched.rsync.complete": [
"bodhi-releng03.%s" % suffix,
],
#FAS messages
topic_prefix + "fas.user.create": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.user.update": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.edit": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.update": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.create": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.role.update": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.member.remove": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.member.sponsor": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
topic_prefix + "fas.group.member.apply": [
"fas-fas0%i.%s" % (i, suffix) for i in range(1, 4)
],
# Git/SCM messages
topic_prefix + "git.receive": [
"scm-pkgs01.%s" % suffix,
],
topic_prefix + "git.lookaside.new": [
"lookaside-pkgs01.%s" % suffix,
],
# Tagger messages
topic_prefix + "fedoratagger.tag.update": [
"fedoratagger-packages0%i.%s" % (i, suffix) for i in range(1, 3)
],
topic_prefix + "fedoratagger.tag.create": [
"fedoratagger-packages0%i.%s" % (i, suffix) for i in range(1, 3)
],
topic_prefix + "fedoratagger.user.rank.update": [
"fedoratagger-packages0%i.%s" % (i, suffix) for i in range(1, 3)
],
# Mediawiki messages
topic_prefix + "wiki.article.edit": [
"mediawiki-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "wiki.upload.complete": [
"mediawiki-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
# Pkgdb messages
topic_prefix + "pkgdb.acl.update": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.acl.request.toggle": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.acl.user.remove": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.owner.update": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.package.new": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.package.update": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.package.retire": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
topic_prefix + "pkgdb.critpath.update": [
"pkgdb-%s" % app_hosts[i-1]
for i in range(1, len(app_hosts) + 1)
],
# Planet/venus
topic_prefix + "planet.post.new": [
"planet-people03.vpn.fedoraproject.org",
],
# Supybot/meetbot
topic_prefix + "meetbot.meeting.start": [
"supybot-value03.%s" % suffix,
],
# Only @spot and @rbergeron can use this one
topic_prefix + "announce.announcement": [
"announce-lockbox01.phx2.fedoraproject.org",
],
},
)

View file

@ -40,6 +40,8 @@
- include: $tasks/rkhunter.yml
- include: $tasks/denyhosts.yml
- include: $tasks/nagios_client.yml
- include: $tasks/fedmsg_base.yml
- include: $tasks/fedmsg_hub.yml
handlers:
- include: $handlers/restart_services.yml

53
tasks/fedmsg-base.yml Normal file
View file

@ -0,0 +1,53 @@
---
# tasklist for setting up fedmsg
# This is the base set of files needed for fedmsg
- name: setup /etc/fedmsg.d directory
file: path=/etc/fedmsg.d owner=root group=root mode=0755
tags:
- config
- name: setup /etc/pki/fedmsg directory
file: path=/etc/pki/fedmsg owner=root group=root mode=0755
tags:
- config
- name: install fedmsg ca.cert
file: source=$puppet_private/fedmsg-certs/keys/ca.crt dest=/etc/pki/fedmsg/ca.crt owner=root group=root mode=0644
tags:
- config
- name: setup /etc/fedmsg.d/ssl.py file
template: src=$files/fedmsg/ssl.py.j2 dest=/etc/fedmsg.d/ssl.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/endpoints.py file
template: src=$files/fedmsg/endpoints.py.j2 dest=/etc/fedmsg.d/endpoints.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/relay.py file
template: src=$files/fedmsg/relay.py.j2 dest=/etc/fedmsg.d/relay.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/pkgdb.py file
template: src=$files/fedmsg/pkgdb.py.j2 dest=/etc/fedmsg.d/pkgdb.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/login.py file
template: src=$files/fedmsg/pkgdb.py.j2 dest=/etc/fedmsg.d/pkgdb.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/logging.py file
template: src=$files/fedmsg/logging.py.j2 dest=/etc/fedmsg.d/logging.py owner=root group=root mode=644
- config
- name: setup /etc/fedmsg.d/base.py file
template: src=$files/fedmsg/base.py.j2 dest=/etc/fedmsg.d/base.py owner=root group=root mode=644
- config
- name: install needed packages
yum: pkg=$item state=installed
with_items:
- fedmsg
tags:
- packages

12
tasks/fedmsg-hub.yml Normal file
View file

@ -0,0 +1,12 @@
---
# Setup a fedmsg-hub
- name: install needed packages
yum: pkg=$item state=installed
with_items:
- fedmsg-hub
tags:
- packages
- name: fedmsg-hub service
service: name=fedmsg-hub state=started enabled=yes