Implement the messaging-bridges app

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>
This commit is contained in:
Aurélien Bompard 2018-09-28 16:32:59 +00:00
parent f9ef6259b3
commit 23291a3b34
7 changed files with 627 additions and 0 deletions

View file

@ -15,3 +15,68 @@
appowners:
- abompard
- jcline
- role: openshift/secret-file
app: messaging-bridges
secret_name: fedmsg-key
key: fedmsg-fedmsg-migration-tools.key
privatefile: fedmsg-certs/keys/messaging-bridges-stg.key
when: env == "staging"
- role: openshift/secret-file
app: messaging-bridges
secret_name: fedmsg-cert
key: fedmsg-fedmsg-migration-tools.crt
privatefile: fedmsg-certs/keys/messaging-bridges-stg.crt
when: env == "staging"
- role: openshift/secret-file
app: messaging-bridges
secret_name: fedmsg-key
key: fedmsg-fedmsg-migration-tools.key
privatefile: fedmsg-certs/keys/messaging-bridges.key
when: env != "staging"
- role: openshift/secret-file
app: messaging-bridges
secret_name: fedmsg-cert
key: fedmsg-fedmsg-migration-tools.crt
privatefile: fedmsg-certs/keys/messaging-bridges.crt
when: env != "staging"
- role: openshift/object
app: messaging-bridges
file: imagestream.yml
objectname: imagestream.yml
- role: openshift/object
app: messaging-bridges
template: buildconfig.yml
objectname: buildconfig.yml
- role: openshift/start-build
app: messaging-bridges
buildname: messaging-bridges-build
- role: openshift/object
app: messaging-bridges
template: configmap.yml
objectname: configmap.yml
- role: openshift/object
app: messaging-bridges
file: service.yml
objectname: service.yml
- role: openshift/object
app: messaging-bridges
template: endpoints.yml
objectname: endpoints.yml
- role: openshift/object
app: messaging-bridges
file: deploymentconfig.yml
objectname: deploymentconfig.yml
- role: openshift/rollout
app: messaging-bridges
dcname: amqp-to-zmq
- role: openshift/rollout
app: messaging-bridges
dcname: zmq-to-amqp
- role: openshift/rollout
app: messaging-bridges
dcname: verify-missing

View file

@ -0,0 +1,216 @@
---
apiVersion: v1
kind: List
metadata: {}
items:
- apiVersion: v1
kind: DeploymentConfig
metadata:
labels:
app: messaging-bridges
name: amqp-to-zmq
spec:
replicas: 1
selector:
deploymentconfig: amqp-to-zmq
strategy:
resources: {}
template:
metadata:
labels:
app: messaging-bridges
deploymentconfig: amqp-to-zmq
spec:
containers:
- name: fedmsg-migration-tools
image: fedmsg-migration-tools:latest
env:
- name: APP_SCRIPT
value: "./.s2i/amqp-to-zmq"
- name: FEDORA_MESSAGING_CONF
value: "/etc/fedora-messaging/amqp-to-zmq.toml"
volumeMounts:
- name: config-volume
mountPath: /etc/fedora-messaging/
readOnly: true
- name: fedmsg-config-volume
mountPath: /etc/fedmsg.d/
readOnly: true
- name: fedmsg-key-volume
mountPath: /etc/pki/fedmsg/key
readOnly: true
- name: fedmsg-crt-volume
mountPath: /etc/pki/fedmsg/crt
readOnly: true
#readinessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 5
# httpGet:
# path: /api/v1/
# port: 8080
#livenessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 30
# httpGet:
# path: /api/v1/
# port: 8080
volumes:
- name: config-volume
configMap:
name: fedora-messaging-configmap
- name: fedmsg-config-volume
configMap:
name: fedmsg-configmap
- name: fedmsg-key-volume
secret:
secretName: fedmsg-key
- name: fedmsg-crt-volume
secret:
secretName: fedmsg-cert
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- fedmsg-migration-tools
from:
kind: ImageStreamTag
name: fedmsg-migration-tools:latest
---
apiVersion: v1
kind: List
metadata: {}
items:
- apiVersion: v1
kind: DeploymentConfig
metadata:
labels:
app: messaging-bridges
name: zmq-to-amqp
spec:
replicas: 1
selector:
deploymentconfig: zmq-to-amqp
strategy:
resources: {}
template:
metadata:
labels:
app: messaging-bridges
deploymentconfig: zmq-to-amqp
spec:
containers:
- name: fedmsg-migration-tools
image: fedmsg-migration-tools:latest
env:
- name: APP_SCRIPT
value: "./.s2i/zmq-to-amqp"
- name: FEDMSG_MIGRATION_TOOLS_CONF
value: "/etc/fedora-messaging/zmq-to-amqp.toml"
volumeMounts:
- name: config-volume
mountPath: /etc/fedora-messaging/
readOnly: true
- name: fedmsg-config-volume
mountPath: /etc/fedmsg.d/
readOnly: true
#readinessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 5
# httpGet:
# path: /api/v1/
# port: 8080
#livenessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 30
# httpGet:
# path: /api/v1/
# port: 8080
volumes:
- name: config-volume
configMap:
name: fedora-messaging-configmap
- name: fedmsg-config-volume
configMap:
name: fedmsg-configmap
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- fedmsg-migration-tools
from:
kind: ImageStreamTag
name: fedmsg-migration-tools:latest
---
apiVersion: v1
kind: List
metadata: {}
items:
- apiVersion: v1
kind: DeploymentConfig
metadata:
labels:
app: messaging-bridges
name: verify-missing
spec:
replicas: 1
selector:
deploymentconfig: verify-missing
strategy:
resources: {}
template:
metadata:
labels:
app: messaging-bridges
deploymentconfig: verify-missing
spec:
containers:
- name: fedmsg-migration-tools
image: fedmsg-migration-tools:latest
env:
- name: APP_SCRIPT
value: "./.s2i/verify-missing"
- name: FEDMSG_MIGRATION_TOOLS_CONF
value: "/etc/fedora-messaging/verify-missing.toml"
volumeMounts:
- name: config-volume
mountPath: /etc/fedora-messaging/
readOnly: true
- name: fedmsg-config-volume
mountPath: /etc/fedmsg.d/
readOnly: true
#readinessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 5
# httpGet:
# path: /api/v1/
# port: 8080
#livenessProbe:
# timeoutSeconds: 1
# initialDelaySeconds: 30
# httpGet:
# path: /api/v1/
# port: 8080
volumes:
- name: config-volume
configMap:
name: fedora-messaging-configmap
- name: fedmsg-config-volume
configMap:
name: fedmsg-configmap
triggers:
- type: ConfigChange
- type: ImageChange
imageChangeParams:
automatic: true
containerNames:
- fedmsg-migration-tools
from:
kind: ImageStreamTag
name: fedmsg-migration-tools:latest

View file

@ -0,0 +1,10 @@
---
apiVersion: v1
kind: ImageStream
metadata:
name: fedmsg-migration-tools
spec:
lookupPolicy:
local: false
status:
dockerImageRepository: ""

View file

@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
labels:
app: messaging-bridges
name: rabbitmq
spec:
ports:
- name: amqp
port: 5672
protocol: TCP
targetPort: 5672
nodePort: 0
selector: {}

View file

@ -0,0 +1,30 @@
apiVersion: v1
kind: BuildConfig
metadata:
name: messaging-bridges-build
labels:
environment: "fedmsg-migration-tools"
spec:
output:
to:
kind: ImageStreamTag
name: fedmsg-migration-tools:latest
source:
type: Git
git:
uri: https://github.com/fedora-infra/fedmsg-migration-tools.git
strategy:
type: Source
sourceStrategy:
from:
kind: ImageStreamTag
namespace: openshift
name: python:3.6
triggers:
{% if fedmsg_migration_tools_github_secret is defined %}
- type: GitHub
github:
secret: "{{ fedmsg_migration_tools_github_secret }}"
{% endif %}
- type: ConfigChange
- type: ImageChange

View file

@ -0,0 +1,272 @@
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fedora-messaging-configmap
labels:
app: messaging-bridges
data:
config.toml: |-
amqp_url = "amqp://rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
publish_exchange = "amq.topic"
#[tls]
#ca_cert = "/etc/pki/tls/certs/ca-bundle.crt"
#keyfile = "/my/client/key.pem"
#certfile = "/my/client/cert.pem"
[client_properties]
app = "fedmsg-migration-tools"
[qos]
prefetch_size = 0
prefetch_count = 25
[log_config]
version = 1
disable_existing_loggers = true
[log_config.formatters.simple]
format = "[%(name)s %(levelname)s] %(message)s"
[log_config.handlers.console]
class = "logging.StreamHandler"
formatter = "simple"
stream = "ext://sys.stdout"
[log_config.loggers.fedora_messaging]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedmsg_migration_tools]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.root]
level = "WARNING"
handlers = ["console"]
amqp-to-zmq.toml: |-
amqp_url = "amqp://rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
publish_exchange = "amq.topic"
[[bindings]]
queue = "amqp_to_zmq"
exchange = "amq.topic"
routing_keys = ["#"]
#[tls]
#ca_cert = "/etc/pki/tls/certs/ca-bundle.crt"
#keyfile = "/my/client/key.pem"
#certfile = "/my/client/cert.pem"
[client_properties]
app = "AMQP to ZMQ"
[queues.amqp_to_zmq]
durable = true
auto_delete = false
exclusive = false
arguments = {}
[qos]
prefetch_size = 0
prefetch_count = 25
[log_config]
version = 1
disable_existing_loggers = true
[log_config.formatters.simple]
format = "[%(name)s %(levelname)s] %(message)s"
[log_config.handlers.console]
class = "logging.StreamHandler"
formatter = "simple"
stream = "ext://sys.stdout"
[log_config.loggers.fedora_messaging]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedmsg]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.root]
level = "WARNING"
handlers = ["console"]
zmq-to-amqp.toml: |-
[zmq_to_amqp]
# The AMQP exchange to publish to.
exchange = "zmq.topic"
# A list of topics to filter the incoming ZMQ messages on; by default all
# messages are forwarded to AMQP.
topics = [""]
# A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket")
zmq_endpoints = [
{% if env == 'staging' %}
"tcp://stg.fedoraproject.org:9940",
{% else %}
"tcp://fedoraproject.org:9940",
{% endif %}
#"tcp://release-monitoring.org:9940",
]
[log_config]
version = 1
disable_existing_loggers = true
[log_config.formatters.simple]
format = "[%(name)s %(levelname)s] %(message)s"
[log_config.handlers.console]
class = "logging.StreamHandler"
formatter = "simple"
stream = "ext://sys.stdout"
[log_config.loggers.fedora_messaging]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedmsg_migration_tools]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedmsg]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.root]
level = "WARNING"
handlers = ["console"]
verify-missing.toml: |-
[zmq_to_amqp]
# The AMQP exchange to publish to.
exchange = "zmq.topic"
# A list of topics to filter the incoming ZMQ messages on; by default all
# messages are forwarded to AMQP.
topics = [""]
# A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket")
zmq_endpoints = [
{% if env == 'staging' %}
"tcp://stg.fedoraproject.org:9940",
{% else %}
"tcp://fedoraproject.org:9940",
{% endif %}
#"tcp://release-monitoring.org:9940",
]
[verify_missing]
# The bindings to use for the AMQP queue
[[verify_missing.bindings]]
exchange = "zmq.topic"
queue = "amqp_bridge_verify_missing"
routing_keys = ["#"]
[[verify_missing.bindings]]
exchange = "amq.topic"
queue = "amqp_bridge_verify_missing"
routing_keys = ["#"]
[queues.verify_missing]
durable = true
auto_delete = false
exclusive = false
arguments = {}
[log_config]
version = 1
disable_existing_loggers = true
[log_config.formatters.simple]
format = "[%(name)s %(levelname)s] %(message)s"
[log_config.handlers.console]
class = "logging.StreamHandler"
formatter = "simple"
stream = "ext://sys.stdout"
[log_config.loggers.fedmsg_migration_tools]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedora_messaging]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.loggers.fedmsg]
level = "INFO"
propagate = false
handlers = ["console"]
[log_config.root]
level = "WARNING"
handlers = ["console"]
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fedmsg-configmap
labels:
app: messaging-bridges
data:
fedmsg-migration-tools.py: |-
# TODO: fedmsg configuration
import socket
hostname = socket.gethostname().split('.', 1)[0]
config = {
'sign_messages': True,
'active': True,
'cert_prefix': 'fedmsg-migration-tools',
'certnames': {'fedmsg-migration-tools.{}'.format(socket.gethostname()): 'fedmsg-migration-tools'},
'relay_inbound': 'tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941',
'bodhi.%s' % hostname: ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
'environment': '{{ env }}',
{% if env == 'staging' %}
# stg should listen to the stg bus
'endpoints': {
'staging_gateway': ['tcp://stg.fedoraproject.org:9940']
},
{% endif %}
"validate_signatures": True,
"crypto_backend": 'x509',
"crypto_validate_backends": ['x509'],
"ssldir": "/etc/pki/fedmsg",
"crl_location": "https://fedoraproject.org/fedmsg/crl.pem",
"crl_cache": "/tmp/fedmsg/crl.pem",
"crl_cache_expiry": 3600,
"ca_cert_location": "https://fedoraproject.org/fedmsg/ca.crt",
"ca_cert_cache": "/tmp/fedmsg/ca.crt",
"ca_cert_cache_expiry": 0, # Never expires
"certnames": {
# In prod/stg, map hostname to the name of the cert in ssldir.
# Unfortunately, we can't use socket.getfqdn()
# "app01.stg": "app01.stg.phx2.fedoraproject.org",
},
# A mapping of fully qualified topics to a list of cert names for which
# a valid signature is to be considered authorized. Messages on topics not
# listed here are considered automatically authorized.
"routing_policy": {
"org.fedoraproject.prod.announce.announcement": [
"announce-lockbox.phx2.fedoraproject.org",
],
},
# Set this to True if you want messages to be dropped that aren't
# explicitly whitelisted in the routing_policy.
# When this is False, only messages that have a topic in the routing_policy
# but whose cert names aren't in the associated list are dropped; messages
# whose topics do not appear in the routing_policy are not dropped.
"routing_nitpicky": False,
# Logging
"logging": {
"version": 1,
"loggers": {
"fedmsg_migration_tools": {
"handlers": ["console"], "propagate": False, "level": "DEBUG"},
"fedmsg": {
"handlers": ["console"], "propagate": False, "level": "DEBUG"},
"moksha": {
"handlers": ["console"], "propagate": False, "level": "DEBUG"},
},
"handlers": {
"console": {
"formatter": "bare",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout",
"level": "DEBUG"
}
},
"formatters": {
"bare": {
"datefmt": "%Y-%m-%d %H:%M:%S",
"format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s"
},
},
},
}

View file

@ -0,0 +1,20 @@
kind: "Endpoints"
apiVersion: "v1"
metadata:
name: "rabbitmq"
subsets:
- addresses:
- ip: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
ports:
- port: 5672
name: "amqp"
- addresses:
- ip: "rabbitmq02{{ env_suffix }}.phx2.fedoraproject.org"
ports:
- port: 5672
name: "amqp"
- addresses:
- ip: "rabbitmq03{{ env_suffix }}.phx2.fedoraproject.org"
ports:
- port: 5672
name: "amqp"