Implement the messaging-bridges app

Signed-off-by: Aurélien Bompard <aurelien@bompard.org>

playbooks/openshift-apps/messaging-bridges.yml

@@ -15,3 +15,68 @@
     appowners:
     - abompard
     - jcline
+
+  - role: openshift/secret-file
+    app: messaging-bridges
+    secret_name: fedmsg-key
+    key: fedmsg-fedmsg-migration-tools.key
+    privatefile: fedmsg-certs/keys/messaging-bridges-stg.key
+    when: env == "staging"
+  - role: openshift/secret-file
+    app: messaging-bridges
+    secret_name: fedmsg-cert
+    key: fedmsg-fedmsg-migration-tools.crt
+    privatefile: fedmsg-certs/keys/messaging-bridges-stg.crt
+    when: env == "staging"
+  - role: openshift/secret-file
+    app: messaging-bridges
+    secret_name: fedmsg-key
+    key: fedmsg-fedmsg-migration-tools.key
+    privatefile: fedmsg-certs/keys/messaging-bridges.key
+    when: env != "staging"
+  - role: openshift/secret-file
+    app: messaging-bridges
+    secret_name: fedmsg-cert
+    key: fedmsg-fedmsg-migration-tools.crt
+    privatefile: fedmsg-certs/keys/messaging-bridges.crt
+    when: env != "staging"
+
+  - role: openshift/object
+    app: messaging-bridges
+    file: imagestream.yml
+    objectname: imagestream.yml
+  - role: openshift/object
+    app: messaging-bridges
+    template: buildconfig.yml
+    objectname: buildconfig.yml
+
+  - role: openshift/start-build
+    app: messaging-bridges
+    buildname: messaging-bridges-build
+
+  - role: openshift/object
+    app: messaging-bridges
+    template: configmap.yml
+    objectname: configmap.yml
+  - role: openshift/object
+    app: messaging-bridges
+    file: service.yml
+    objectname: service.yml
+  - role: openshift/object
+    app: messaging-bridges
+    template: endpoints.yml
+    objectname: endpoints.yml
+  - role: openshift/object
+    app: messaging-bridges
+    file: deploymentconfig.yml
+    objectname: deploymentconfig.yml
+
+  - role: openshift/rollout
+    app: messaging-bridges
+    dcname: amqp-to-zmq
+  - role: openshift/rollout
+    app: messaging-bridges
+    dcname: zmq-to-amqp
+  - role: openshift/rollout
+    app: messaging-bridges
+    dcname: verify-missing

roles/openshift-apps/messaging-bridges/files/deploymentconfig.yml

@@ -0,0 +1,216 @@
+---
+apiVersion: v1
+kind: List
+metadata: {}
+items:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    labels:
+      app: messaging-bridges
+    name: amqp-to-zmq
+  spec:
+    replicas: 1
+    selector:
+      deploymentconfig: amqp-to-zmq
+    strategy:
+      resources: {}
+    template:
+      metadata:
+        labels:
+          app: messaging-bridges
+          deploymentconfig: amqp-to-zmq
+      spec:
+        containers:
+        - name: fedmsg-migration-tools
+          image: fedmsg-migration-tools:latest
+          env:
+          - name: APP_SCRIPT
+            value: "./.s2i/amqp-to-zmq"
+          - name: FEDORA_MESSAGING_CONF
+            value: "/etc/fedora-messaging/amqp-to-zmq.toml"
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/fedora-messaging/
+              readOnly: true
+            - name: fedmsg-config-volume
+              mountPath: /etc/fedmsg.d/
+              readOnly: true
+            - name: fedmsg-key-volume
+              mountPath: /etc/pki/fedmsg/key
+              readOnly: true
+            - name: fedmsg-crt-volume
+              mountPath: /etc/pki/fedmsg/crt
+              readOnly: true
+          #readinessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 5
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+          #livenessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 30
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+        volumes:
+          - name: config-volume
+            configMap:
+              name: fedora-messaging-configmap
+          - name: fedmsg-config-volume
+            configMap:
+              name: fedmsg-configmap
+          - name: fedmsg-key-volume
+            secret:
+              secretName: fedmsg-key
+          - name: fedmsg-crt-volume
+            secret:
+              secretName: fedmsg-cert
+
+    triggers:
+    - type: ConfigChange
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+        - fedmsg-migration-tools
+        from:
+          kind: ImageStreamTag
+          name: fedmsg-migration-tools:latest
+---
+apiVersion: v1
+kind: List
+metadata: {}
+items:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    labels:
+      app: messaging-bridges
+    name: zmq-to-amqp
+  spec:
+    replicas: 1
+    selector:
+      deploymentconfig: zmq-to-amqp
+    strategy:
+      resources: {}
+    template:
+      metadata:
+        labels:
+          app: messaging-bridges
+          deploymentconfig: zmq-to-amqp
+      spec:
+        containers:
+        - name: fedmsg-migration-tools
+          image: fedmsg-migration-tools:latest
+          env:
+          - name: APP_SCRIPT
+            value: "./.s2i/zmq-to-amqp"
+          - name: FEDMSG_MIGRATION_TOOLS_CONF
+            value: "/etc/fedora-messaging/zmq-to-amqp.toml"
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/fedora-messaging/
+              readOnly: true
+            - name: fedmsg-config-volume
+              mountPath: /etc/fedmsg.d/
+              readOnly: true
+          #readinessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 5
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+          #livenessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 30
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+        volumes:
+          - name: config-volume
+            configMap:
+              name: fedora-messaging-configmap
+          - name: fedmsg-config-volume
+            configMap:
+              name: fedmsg-configmap
+
+    triggers:
+    - type: ConfigChange
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+        - fedmsg-migration-tools
+        from:
+          kind: ImageStreamTag
+          name: fedmsg-migration-tools:latest
+---
+apiVersion: v1
+kind: List
+metadata: {}
+items:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    labels:
+      app: messaging-bridges
+    name: verify-missing
+  spec:
+    replicas: 1
+    selector:
+      deploymentconfig: verify-missing
+    strategy:
+      resources: {}
+    template:
+      metadata:
+        labels:
+          app: messaging-bridges
+          deploymentconfig: verify-missing
+      spec:
+        containers:
+        - name: fedmsg-migration-tools
+          image: fedmsg-migration-tools:latest
+          env:
+          - name: APP_SCRIPT
+            value: "./.s2i/verify-missing"
+          - name: FEDMSG_MIGRATION_TOOLS_CONF
+            value: "/etc/fedora-messaging/verify-missing.toml"
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/fedora-messaging/
+              readOnly: true
+            - name: fedmsg-config-volume
+              mountPath: /etc/fedmsg.d/
+              readOnly: true
+          #readinessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 5
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+          #livenessProbe:
+          #  timeoutSeconds: 1
+          #  initialDelaySeconds: 30
+          #  httpGet:
+          #    path: /api/v1/
+          #    port: 8080
+        volumes:
+          - name: config-volume
+            configMap:
+              name: fedora-messaging-configmap
+          - name: fedmsg-config-volume
+            configMap:
+              name: fedmsg-configmap
+
+    triggers:
+    - type: ConfigChange
+    - type: ImageChange
+      imageChangeParams:
+        automatic: true
+        containerNames:
+        - fedmsg-migration-tools
+        from:
+          kind: ImageStreamTag
+          name: fedmsg-migration-tools:latest

roles/openshift-apps/messaging-bridges/files/imagestream.yml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: ImageStream
+metadata:
+  name: fedmsg-migration-tools 
+spec:
+  lookupPolicy:
+    local: false
+status:
+  dockerImageRepository: ""

roles/openshift-apps/messaging-bridges/files/service.yml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: messaging-bridges
+  name: rabbitmq
+spec:
+  ports:
+  - name: amqp
+    port: 5672
+    protocol: TCP
+    targetPort: 5672
+    nodePort: 0
+  selector: {}

roles/openshift-apps/messaging-bridges/templates/buildconfig.yml

@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: BuildConfig
+metadata:
+  name: messaging-bridges-build
+  labels:
+    environment: "fedmsg-migration-tools"
+spec:
+  output:
+    to:
+      kind: ImageStreamTag
+      name: fedmsg-migration-tools:latest
+  source:
+    type: Git
+    git:
+      uri: https://github.com/fedora-infra/fedmsg-migration-tools.git
+  strategy:
+    type: Source
+    sourceStrategy:
+      from:
+        kind: ImageStreamTag
+        namespace: openshift
+        name: python:3.6
+  triggers:
+{% if fedmsg_migration_tools_github_secret is defined %}
+  - type: GitHub
+    github:
+      secret: "{{ fedmsg_migration_tools_github_secret }}"
+{% endif %}
+  - type: ConfigChange
+  - type: ImageChange

roles/openshift-apps/messaging-bridges/templates/configmap.yml

@@ -0,0 +1,272 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fedora-messaging-configmap
+  labels:
+    app: messaging-bridges
+data:
+  config.toml: |-
+    amqp_url = "amqp://rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+    publish_exchange = "amq.topic"
+    
+    #[tls]
+    #ca_cert = "/etc/pki/tls/certs/ca-bundle.crt"
+    #keyfile = "/my/client/key.pem"
+    #certfile = "/my/client/cert.pem"
+    
+    [client_properties]
+    app = "fedmsg-migration-tools"
+    
+    [qos]
+    prefetch_size = 0
+    prefetch_count = 25
+    
+    [log_config]
+    version = 1
+    disable_existing_loggers = true
+    [log_config.formatters.simple]
+    format = "[%(name)s %(levelname)s] %(message)s"
+    [log_config.handlers.console]
+    class = "logging.StreamHandler"
+    formatter = "simple"
+    stream = "ext://sys.stdout"
+    [log_config.loggers.fedora_messaging]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedmsg_migration_tools]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.root]
+    level = "WARNING"
+    handlers = ["console"]
+  amqp-to-zmq.toml: |-
+    amqp_url = "amqp://rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+    publish_exchange = "amq.topic"
+    
+    [[bindings]]
+    queue = "amqp_to_zmq"
+    exchange = "amq.topic"
+    routing_keys = ["#"]
+    
+    #[tls]
+    #ca_cert = "/etc/pki/tls/certs/ca-bundle.crt"
+    #keyfile = "/my/client/key.pem"
+    #certfile = "/my/client/cert.pem"
+    
+    [client_properties]
+    app = "AMQP to ZMQ"
+    
+    [queues.amqp_to_zmq]
+    durable = true
+    auto_delete = false
+    exclusive = false
+    arguments = {}
+    
+    [qos]
+    prefetch_size = 0
+    prefetch_count = 25
+    
+    [log_config]
+    version = 1
+    disable_existing_loggers = true
+    [log_config.formatters.simple]
+    format = "[%(name)s %(levelname)s] %(message)s"
+    [log_config.handlers.console]
+    class = "logging.StreamHandler"
+    formatter = "simple"
+    stream = "ext://sys.stdout"
+    [log_config.loggers.fedora_messaging]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedmsg]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.root]
+    level = "WARNING"
+    handlers = ["console"]
+  zmq-to-amqp.toml: |-
+    [zmq_to_amqp]
+    # The AMQP exchange to publish to.
+    exchange = "zmq.topic"
+    # A list of topics to filter the incoming ZMQ messages on; by default all
+    # messages are forwarded to AMQP.
+    topics = [""]
+    # A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket")
+    zmq_endpoints = [
+{% if env == 'staging' %}
+        "tcp://stg.fedoraproject.org:9940",
+{% else %}
+        "tcp://fedoraproject.org:9940",
+{% endif %}
+        #"tcp://release-monitoring.org:9940",
+    ]
+    
+    [log_config]
+    version = 1
+    disable_existing_loggers = true
+    [log_config.formatters.simple]
+    format = "[%(name)s %(levelname)s] %(message)s"
+    [log_config.handlers.console]
+    class = "logging.StreamHandler"
+    formatter = "simple"
+    stream = "ext://sys.stdout"
+    [log_config.loggers.fedora_messaging]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedmsg_migration_tools]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedmsg]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.root]
+    level = "WARNING"
+    handlers = ["console"]
+  verify-missing.toml: |-
+    [zmq_to_amqp]
+    # The AMQP exchange to publish to.
+    exchange = "zmq.topic"
+    # A list of topics to filter the incoming ZMQ messages on; by default all
+    # messages are forwarded to AMQP.
+    topics = [""]
+    # A list of strings in the ZeroMQ socket format (e.g. "tcp://host:socket")
+    zmq_endpoints = [
+{% if env == 'staging' %}
+        "tcp://stg.fedoraproject.org:9940",
+{% else %}
+        "tcp://fedoraproject.org:9940",
+{% endif %}
+        #"tcp://release-monitoring.org:9940",
+    ]
+    
+    [verify_missing]
+    # The bindings to use for the AMQP queue
+    [[verify_missing.bindings]]
+    exchange = "zmq.topic"
+    queue = "amqp_bridge_verify_missing"
+    routing_keys = ["#"]
+    [[verify_missing.bindings]]
+    exchange = "amq.topic"
+    queue = "amqp_bridge_verify_missing"
+    routing_keys = ["#"]
+
+    [queues.verify_missing]
+    durable = true
+    auto_delete = false
+    exclusive = false
+    arguments = {}
+    
+    [log_config]
+    version = 1
+    disable_existing_loggers = true
+    [log_config.formatters.simple]
+    format = "[%(name)s %(levelname)s] %(message)s"
+    [log_config.handlers.console]
+    class = "logging.StreamHandler"
+    formatter = "simple"
+    stream = "ext://sys.stdout"
+    [log_config.loggers.fedmsg_migration_tools]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedora_messaging]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.loggers.fedmsg]
+    level = "INFO"
+    propagate = false
+    handlers = ["console"]
+    [log_config.root]
+    level = "WARNING"
+    handlers = ["console"]
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fedmsg-configmap
+  labels:
+    app: messaging-bridges
+data:
+  fedmsg-migration-tools.py: |-
+    # TODO: fedmsg configuration
+    import socket
+    hostname = socket.gethostname().split('.', 1)[0]
+    config = {
+        'sign_messages': True,
+        'active': True,
+        'cert_prefix': 'fedmsg-migration-tools',
+        'certnames': {'fedmsg-migration-tools.{}'.format(socket.gethostname()): 'fedmsg-migration-tools'},
+        'relay_inbound': 'tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941',
+        'bodhi.%s' % hostname: ['tcp://busgateway01{{ env_suffix }}.phx2.fedoraproject.org:9941'],
+        'environment': '{{ env }}',
+{% if env == 'staging' %}
+        # stg should listen to the stg bus
+        'endpoints': {
+            'staging_gateway': ['tcp://stg.fedoraproject.org:9940']
+        },
+{% endif %}
+        "validate_signatures": True,
+        "crypto_backend": 'x509',
+        "crypto_validate_backends": ['x509'],
+        "ssldir": "/etc/pki/fedmsg",
+        "crl_location": "https://fedoraproject.org/fedmsg/crl.pem",
+        "crl_cache": "/tmp/fedmsg/crl.pem",
+        "crl_cache_expiry": 3600,
+        "ca_cert_location": "https://fedoraproject.org/fedmsg/ca.crt",
+        "ca_cert_cache": "/tmp/fedmsg/ca.crt",
+        "ca_cert_cache_expiry": 0,  # Never expires
+        "certnames": {
+            # In prod/stg, map hostname to the name of the cert in ssldir.
+            # Unfortunately, we can't use socket.getfqdn()
+            # "app01.stg": "app01.stg.phx2.fedoraproject.org",
+        },
+        # A mapping of fully qualified topics to a list of cert names for which
+        # a valid signature is to be considered authorized.  Messages on topics not
+        # listed here are considered automatically authorized.
+        "routing_policy": {
+            "org.fedoraproject.prod.announce.announcement": [
+                "announce-lockbox.phx2.fedoraproject.org",
+            ],
+        },
+        # Set this to True if you want messages to be dropped that aren't
+        # explicitly whitelisted in the routing_policy.
+        # When this is False, only messages that have a topic in the routing_policy
+        # but whose cert names aren't in the associated list are dropped; messages
+        # whose topics do not appear in the routing_policy are not dropped.
+        "routing_nitpicky": False,
+        # Logging
+        "logging": {
+            "version": 1,
+            "loggers": {
+                "fedmsg_migration_tools": {
+                    "handlers": ["console"], "propagate": False, "level": "DEBUG"},
+                "fedmsg": {
+                    "handlers": ["console"], "propagate": False, "level": "DEBUG"},
+                "moksha": {
+                    "handlers": ["console"], "propagate": False, "level": "DEBUG"},
+            },
+            "handlers": {
+                "console": {
+                    "formatter": "bare",
+                    "class": "logging.StreamHandler",
+                    "stream": "ext://sys.stdout",
+                    "level": "DEBUG"
+                }
+            },
+            "formatters": {
+                "bare": {
+                    "datefmt": "%Y-%m-%d %H:%M:%S",
+                    "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s"
+                },
+            },
+        },
+    }

roles/openshift-apps/messaging-bridges/templates/endpoints.yml

@@ -0,0 +1,20 @@
+kind: "Endpoints"
+apiVersion: "v1"
+metadata:
+  name: "rabbitmq"
+subsets:
+  - addresses:
+    - ip: "rabbitmq01{{ env_suffix }}.phx2.fedoraproject.org"
+    ports:
+    - port: 5672
+      name: "amqp"
+  - addresses:
+    - ip: "rabbitmq02{{ env_suffix }}.phx2.fedoraproject.org"
+    ports:
+    - port: 5672
+      name: "amqp"
+  - addresses:
+    - ip: "rabbitmq03{{ env_suffix }}.phx2.fedoraproject.org"
+    ports:
+    - port: 5672
+      name: "amqp"