diff --git a/roles/openshift-apps/messaging-bridges/templates/amqp-to-zmq.toml b/roles/openshift-apps/messaging-bridges/templates/amqp-to-zmq.toml new file mode 100644 index 0000000000..3bbd974949 --- /dev/null +++ b/roles/openshift-apps/messaging-bridges/templates/amqp-to-zmq.toml @@ -0,0 +1,67 @@ +amqp_url = "amqps://messaging-bridge{{ env_suffix }}.fedoraproject.org:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" +publish_exchange = "amq.topic" +passive_declares = true + +[[bindings]] +queue = "amqp_to_zmq" +exchange = "amq.topic" +routing_keys = ["#"] + +[tls] +ca_cert = "/etc/pki/rabbitmq/ca/rabbitmq-ca.crt" +keyfile = "/etc/pki/rabbitmq/key/rabbitmq-fedmsg-migration-tools.key" +certfile = "/etc/pki/rabbitmq/crt/rabbitmq-fedmsg-migration-tools.crt" + +[client_properties] +app = "AMQP to ZMQ" + +[queues.amqp_to_zmq] +durable = true +auto_delete = false +exclusive = false +arguments = {} + +[consumer_config] +publish_endpoint = "tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941" +remote_publish = true + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true +[log_config.filters.ratelimit] +"()" = "fedmsg_migration_tools.filters.RateLimiter" +rate = 120 +[log_config.formatters.simple] +format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +filters = ["ratelimit"] +formatter = "simple" +mailhost = "bastion" +fromaddr = "fedmsg-migration-tools@fedoraproject.org" +toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] +subject = "ERROR: fedmsg-migration-tools amqp-to-zmq bridge" +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg_migration_tools] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.root] +level = "WARNING" +handlers = ["console", "email"] diff --git a/roles/openshift-apps/messaging-bridges/templates/config.toml b/roles/openshift-apps/messaging-bridges/templates/config.toml new file mode 100644 index 0000000000..93904a335f --- /dev/null +++ b/roles/openshift-apps/messaging-bridges/templates/config.toml @@ -0,0 +1,37 @@ +amqp_url = "amqps://messaging-bridge{{ env_suffix }}.fedoraproject.org:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" +publish_exchange = "amq.topic" +passive_declares = true + +[tls] +ca_cert = "/etc/pki/rabbitmq/ca/rabbitmq-ca.crt" +keyfile = "/etc/pki/rabbitmq/key/rabbitmq-fedmsg-migration-tools.key" +certfile = "/etc/pki/rabbitmq/crt/rabbitmq-fedmsg-migration-tools.crt" + +[client_properties] +app = "fedmsg-migration-tools" + +[qos] +prefetch_size = 0 +prefetch_count = 25 + +[log_config] +version = 1 +disable_existing_loggers = true +[log_config.formatters.simple] +format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg_migration_tools] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.root] +level = "WARNING" +handlers = ["console"] diff --git a/roles/openshift-apps/messaging-bridges/templates/configmap.yml b/roles/openshift-apps/messaging-bridges/templates/configmap.yml index eb578e943e..46f6aef486 100644 --- a/roles/openshift-apps/messaging-bridges/templates/configmap.yml +++ b/roles/openshift-apps/messaging-bridges/templates/configmap.yml @@ -1,3 +1,4 @@ +{% macro load_file(filename) %}{% include filename %}{%- endmacro -%} --- apiVersion: v1 kind: ConfigMap @@ -7,244 +8,13 @@ metadata: app: messaging-bridges data: config.toml: |- - amqp_url = "amqps://messaging-bridge{{ env_suffix }}.fedoraproject.org:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" - publish_exchange = "amq.topic" - passive_declares = true - - [tls] - ca_cert = "/etc/pki/rabbitmq/ca/rabbitmq-ca.crt" - keyfile = "/etc/pki/rabbitmq/key/rabbitmq-fedmsg-migration-tools.key" - certfile = "/etc/pki/rabbitmq/crt/rabbitmq-fedmsg-migration-tools.crt" - - [client_properties] - app = "fedmsg-migration-tools" - - [qos] - prefetch_size = 0 - prefetch_count = 25 - - [log_config] - version = 1 - disable_existing_loggers = true - [log_config.formatters.simple] - format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" - datefmt = "%Y-%m-%d %H:%M:%S" - [log_config.handlers.console] - class = "logging.StreamHandler" - formatter = "simple" - stream = "ext://sys.stdout" - [log_config.loggers.fedora_messaging] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg_migration_tools] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.root] - level = "WARNING" - handlers = ["console"] + {{ load_file('config.toml') | indent }} amqp-to-zmq.toml: |- - amqp_url = "amqps://messaging-bridge{{ env_suffix }}.fedoraproject.org:@rabbitmq{{ env_suffix }}.fedoraproject.org/%2Fpubsub" - publish_exchange = "amq.topic" - passive_declares = true - - [[bindings]] - queue = "amqp_to_zmq" - exchange = "amq.topic" - routing_keys = ["#"] - - [tls] - ca_cert = "/etc/pki/rabbitmq/ca/rabbitmq-ca.crt" - keyfile = "/etc/pki/rabbitmq/key/rabbitmq-fedmsg-migration-tools.key" - certfile = "/etc/pki/rabbitmq/crt/rabbitmq-fedmsg-migration-tools.crt" - - [client_properties] - app = "AMQP to ZMQ" - - [queues.amqp_to_zmq] - durable = true - auto_delete = false - exclusive = false - arguments = {} - - [consumer_config] - publish_endpoint = "tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941" - remote_publish = true - - [qos] - prefetch_size = 0 - prefetch_count = 25 - - [log_config] - version = 1 - disable_existing_loggers = true - [log_config.filters.ratelimit] - "()" = "fedmsg_migration_tools.filters.RateLimiter" - rate = 120 - [log_config.formatters.simple] - format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" - datefmt = "%Y-%m-%d %H:%M:%S" - [log_config.handlers.console] - class = "logging.StreamHandler" - formatter = "simple" - stream = "ext://sys.stdout" - [log_config.handlers.email] - class = "logging.handlers.SMTPHandler" - filters = ["ratelimit"] - formatter = "simple" - mailhost = "bastion" - fromaddr = "fedmsg-migration-tools@fedoraproject.org" - toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] - subject = "ERROR: fedmsg-migration-tools amqp-to-zmq bridge" - [log_config.loggers.fedora_messaging] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg_migration_tools] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.root] - level = "WARNING" - handlers = ["console", "email"] + {{ load_file('amqp-to-zmq.toml') | indent }} zmq-to-amqp.toml: |- - [zmq_to_amqp] - # The AMQP exchange to publish to. - exchange = "zmq.topic" - # A list of topics to filter the incoming ZMQ messages on; by default all - # messages are forwarded to AMQP. - topics = [""] - # A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket") - zmq_endpoints = [ -{% if env == 'staging' %} - "tcp://stg.fedoraproject.org:9940", -{% else %} - "tcp://fedoraproject.org:9940", -{% endif %} - #"tcp://release-monitoring.org:9940", - ] - - [log_config] - version = 1 - disable_existing_loggers = true - [log_config.filters.ratelimit] - "()" = "fedmsg_migration_tools.filters.RateLimiter" - rate = 120 - [log_config.formatters.simple] - format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" - datefmt = "%Y-%m-%d %H:%M:%S" - [log_config.handlers.console] - class = "logging.StreamHandler" - formatter = "simple" - stream = "ext://sys.stdout" - [log_config.handlers.email] - class = "logging.handlers.SMTPHandler" - filters = ["ratelimit"] - formatter = "simple" - mailhost = "bastion" - fromaddr = "fedmsg-migration-tools@fedoraproject.org" - toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] - subject = "ERROR: fedmsg-migration-tools zmq-to-amqp bridge" - [log_config.loggers.fedora_messaging] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg_migration_tools] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.root] - level = "WARNING" - handlers = ["console", "email"] + {{ load_file('zmq-to-amqp.toml') | indent }} verify-missing.toml: |- - [zmq_to_amqp] - # The AMQP exchange to publish to. - exchange = "zmq.topic" - # A list of topics to filter the incoming ZMQ messages on; by default all - # messages are forwarded to AMQP. - topics = [""] - # A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket") - zmq_endpoints = [ -{% if env == 'staging' %} - "tcp://stg.fedoraproject.org:9940", -{% else %} - "tcp://fedoraproject.org:9940", -{% endif %} - #"tcp://release-monitoring.org:9940", - ] - - [verify_missing] - # The queue to setup - [verify_missing.queue] - queue = "amqp_bridge_verify_missing" - durable = true - auto_delete = false - exclusive = false - arguments = { x-message-ttl = 60000 } - # The exchanges to declare - [[verify_missing.exchanges]] - exchange = "amq.topic" - exchange_type = "topic" - durable = true - [[verify_missing.exchanges]] - exchange = "zmq.topic" - exchange_type = "topic" - durable = true - # The bindings to use for the AMQP queue - [[verify_missing.bindings]] - exchange = "zmq.topic" - queue = "amqp_bridge_verify_missing" - routing_key = "#" - [[verify_missing.bindings]] - exchange = "amq.topic" - queue = "amqp_bridge_verify_missing" - routing_key = "#" - - [log_config] - version = 1 - disable_existing_loggers = true - [log_config.filters.ratelimit] - "()" = "fedmsg_migration_tools.filters.RateLimiter" - rate = 120 - [log_config.formatters.simple] - format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" - datefmt = "%Y-%m-%d %H:%M:%S" - [log_config.handlers.console] - class = "logging.StreamHandler" - formatter = "simple" - stream = "ext://sys.stdout" - [log_config.handlers.email] - class = "logging.handlers.SMTPHandler" - formatter = "simple" - filters = ["ratelimit"] - mailhost = "bastion" - fromaddr = "fedmsg-migration-tools@fedoraproject.org" - toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] - subject = "ERROR: fedmsg-migration-tools verify-missing service ({{ env }})" - [log_config.loggers.fedmsg_migration_tools] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedora_messaging] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.loggers.fedmsg] - level = "INFO" - propagate = false - handlers = ["console"] - [log_config.root] - level = "WARNING" - handlers = ["console", "email"] + {{ load_file('verify-missing.toml') | indent }} --- apiVersion: v1 kind: ConfigMap @@ -254,78 +24,4 @@ metadata: app: messaging-bridges data: fedmsg-migration-tools.py: |- - import socket - hostname = socket.gethostname().split('.', 1)[0] - config = { - 'sign_messages': True, - 'active': True, - 'cert_prefix': 'fedmsg-migration-tools', - 'certnames': {'fedmsg-migration-tools.{}'.format(socket.gethostname()): 'fedmsg-migration-tools'}, - 'relay_inbound': 'tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941', -{% if env == 'staging' %} - # stg should listen to the stg bus - 'endpoints': { - 'staging_gateway': ['tcp://stg.fedoraproject.org:9940'] - }, - 'environment': 'stg', -{% else %} - 'environment': 'prod', -{% endif %} - "validate_signatures": True, - "crypto_backend": 'x509', - "crypto_validate_backends": ['x509'], - "ssldir": "/opt/app-root/etc/pki/fedmsg", - "crl_location": "https://fedoraproject.org/fedmsg/crl.pem", - "crl_cache": "/tmp/fedmsg/crl.pem", - "crl_cache_expiry": 3600, - "ca_cert_location": "https://fedoraproject.org/fedmsg/ca.crt", - "ca_cert_cache": "/tmp/fedmsg/ca.crt", - "ca_cert_cache_expiry": 0, # Never expires - # A mapping of fully qualified topics to a list of cert names for which - # a valid signature is to be considered authorized. Messages on topics not - # listed here are considered automatically authorized. - # ** policy dynamically generated from inventory vars - # See ansible/filter_plugins/fedmsg.py for this inversion filter. - "routing_policy": { - {% for topic, certs in groups | invert_fedmsg_policy(hostvars, env) %} - "{{topic}}": [ - {% for cert in certs %} - "{{ cert }}", - {% endfor %} - "fedmsg-migration-tools-fedmsg-migration-tools{{env_suffix}}.fedoraproject.org", - ], - {% endfor %} - }, - # Set this to True if you want messages to be dropped that aren't - # explicitly whitelisted in the routing_policy. - # When this is False, only messages that have a topic in the routing_policy - # but whose cert names aren't in the associated list are dropped; messages - # whose topics do not appear in the routing_policy are not dropped. - "routing_nitpicky": False, - # Logging - "logging": { - "version": 1, - "loggers": { - "fedmsg_migration_tools": { - "handlers": ["console"], "propagate": False, "level": "DEBUG"}, - "fedmsg": { - "handlers": ["console"], "propagate": False, "level": "DEBUG"}, - "moksha": { - "handlers": ["console"], "propagate": False, "level": "DEBUG"}, - }, - "handlers": { - "console": { - "formatter": "bare", - "class": "logging.StreamHandler", - "stream": "ext://sys.stdout", - "level": "DEBUG" - } - }, - "formatters": { - "bare": { - "datefmt": "%Y-%m-%d %H:%M:%S", - "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" - }, - }, - }, - } + {{ load_file('fedmsg-migration-tools.py') | indent }} diff --git a/roles/openshift-apps/messaging-bridges/templates/fedmsg-migration-tools.py b/roles/openshift-apps/messaging-bridges/templates/fedmsg-migration-tools.py new file mode 100644 index 0000000000..49ba7b5ed5 --- /dev/null +++ b/roles/openshift-apps/messaging-bridges/templates/fedmsg-migration-tools.py @@ -0,0 +1,76 @@ +import socket + +hostname = socket.gethostname().split('.', 1)[0] +config = { + 'sign_messages': True, + 'active': True, + 'cert_prefix': 'fedmsg-migration-tools', + 'certnames': {'fedmsg-migration-tools.{}'.format(socket.gethostname()): 'fedmsg-migration-tools'}, + 'relay_inbound': 'tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941', +{% if env == 'staging' %} +# stg should listen to the stg bus + 'endpoints': { + 'staging_gateway': ['tcp://stg.fedoraproject.org:9940'] + }, + 'environment': 'stg', +{% else %} + 'environment': 'prod', +{% endif %} + "validate_signatures": True, + "crypto_backend": 'x509', + "crypto_validate_backends": ['x509'], + "ssldir": "/opt/app-root/etc/pki/fedmsg", + "crl_location": "https://fedoraproject.org/fedmsg/crl.pem", + "crl_cache": "/tmp/fedmsg/crl.pem", + "crl_cache_expiry": 3600, + "ca_cert_location": "https://fedoraproject.org/fedmsg/ca.crt", + "ca_cert_cache": "/tmp/fedmsg/ca.crt", + "ca_cert_cache_expiry": 0, # Never expires + # A mapping of fully qualified topics to a list of cert names for which + # a valid signature is to be considered authorized. Messages on topics not + # listed here are considered automatically authorized. + # ** policy dynamically generated from inventory vars + # See ansible/filter_plugins/fedmsg.py for this inversion filter. + "routing_policy": { + {% for topic, certs in groups | invert_fedmsg_policy(hostvars, env) %} + "{{topic}}": [ + {% for cert in certs %} + "{{ cert }}", +{% endfor %} +"fedmsg-migration-tools-fedmsg-migration-tools{{env_suffix}}.fedoraproject.org", + ], +{% endfor %} + }, + # Set this to True if you want messages to be dropped that aren't + # explicitly whitelisted in the routing_policy. + # When this is False, only messages that have a topic in the routing_policy + # but whose cert names aren't in the associated list are dropped; messages + # whose topics do not appear in the routing_policy are not dropped. + "routing_nitpicky": False, + # Logging + "logging": { + "version": 1, + "loggers": { + "fedmsg_migration_tools": { + "handlers": ["console"], "propagate": False, "level": "DEBUG"}, + "fedmsg": { + "handlers": ["console"], "propagate": False, "level": "DEBUG"}, + "moksha": { + "handlers": ["console"], "propagate": False, "level": "DEBUG"}, + }, + "handlers": { + "console": { + "formatter": "bare", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout", + "level": "DEBUG" + } + }, + "formatters": { + "bare": { + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" + }, + }, + }, +} diff --git a/roles/openshift-apps/messaging-bridges/templates/verify-missing.toml b/roles/openshift-apps/messaging-bridges/templates/verify-missing.toml new file mode 100644 index 0000000000..be238d5e60 --- /dev/null +++ b/roles/openshift-apps/messaging-bridges/templates/verify-missing.toml @@ -0,0 +1,79 @@ +[zmq_to_amqp] +# The AMQP exchange to publish to. +exchange = "zmq.topic" +# A list of topics to filter the incoming ZMQ messages on; by default all +# messages are forwarded to AMQP. +topics = [""] +# A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket") +zmq_endpoints = [ +{% if env == 'staging' %} + "tcp://stg.fedoraproject.org:9940", +{% else %} + "tcp://fedoraproject.org:9940", +{% endif %} + #"tcp://release-monitoring.org:9940", +] + +[verify_missing] +# The queue to setup +[verify_missing.queue] +queue = "amqp_bridge_verify_missing" +durable = true +auto_delete = false +exclusive = false +arguments = { x-message-ttl = 60000 } +# The exchanges to declare +[[verify_missing.exchanges]] +exchange = "amq.topic" +exchange_type = "topic" +durable = true +[[verify_missing.exchanges]] +exchange = "zmq.topic" +exchange_type = "topic" +durable = true +# The bindings to use for the AMQP queue +[[verify_missing.bindings]] +exchange = "zmq.topic" +queue = "amqp_bridge_verify_missing" +routing_key = "#" +[[verify_missing.bindings]] +exchange = "amq.topic" +queue = "amqp_bridge_verify_missing" +routing_key = "#" + +[log_config] +version = 1 +disable_existing_loggers = true +[log_config.filters.ratelimit] +"()" = "fedmsg_migration_tools.filters.RateLimiter" +rate = 120 +[log_config.formatters.simple] +format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +formatter = "simple" +filters = ["ratelimit"] +mailhost = "bastion" +fromaddr = "fedmsg-migration-tools@fedoraproject.org" +toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] +subject = "ERROR: fedmsg-migration-tools verify-missing service ({{ env }})" +[log_config.loggers.fedmsg_migration_tools] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.root] +level = "WARNING" +handlers = ["console", "email"] diff --git a/roles/openshift-apps/messaging-bridges/templates/zmq-to-amqp.toml b/roles/openshift-apps/messaging-bridges/templates/zmq-to-amqp.toml new file mode 100644 index 0000000000..fea4655851 --- /dev/null +++ b/roles/openshift-apps/messaging-bridges/templates/zmq-to-amqp.toml @@ -0,0 +1,52 @@ +[zmq_to_amqp] +# The AMQP exchange to publish to. +exchange = "zmq.topic" +# A list of topics to filter the incoming ZMQ messages on; by default all +# messages are forwarded to AMQP. +topics = [""] +# A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket") +zmq_endpoints = [ +{% if env == 'staging' %} + "tcp://stg.fedoraproject.org:9940", +{% else %} + "tcp://fedoraproject.org:9940", +{% endif %} + #"tcp://release-monitoring.org:9940", +] + +[log_config] +version = 1 +disable_existing_loggers = true +[log_config.filters.ratelimit] +"()" = "fedmsg_migration_tools.filters.RateLimiter" +rate = 120 +[log_config.formatters.simple] +format = "%(asctime)s [%(name)s %(levelname)s] %(message)s" +datefmt = "%Y-%m-%d %H:%M:%S" +[log_config.handlers.console] +class = "logging.StreamHandler" +formatter = "simple" +stream = "ext://sys.stdout" +[log_config.handlers.email] +class = "logging.handlers.SMTPHandler" +filters = ["ratelimit"] +formatter = "simple" +mailhost = "bastion" +fromaddr = "fedmsg-migration-tools@fedoraproject.org" +toaddrs = ["jcline@fedoraproject.org", "abompard@fedoraproject.org"] +subject = "ERROR: fedmsg-migration-tools zmq-to-amqp bridge" +[log_config.loggers.fedora_messaging] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg_migration_tools] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.loggers.fedmsg] +level = "INFO" +propagate = false +handlers = ["console"] +[log_config.root] +level = "WARNING" +handlers = ["console", "email"]