fedora-messaging-migration: Patches from Karsten Hopp to move script calling things from fedmsg to fedora-messaging

We will need to check these services after pushing this out and confirm
that they are still emitting or hearing messages they need to.

Many thanks Karsten!

callback_plugins/fedora_messaging_callback.py

@@ -0,0 +1,103 @@
+# (C) 2012, Michael DeHaan, <michael.dehaan@gmail.com>
+# based on the log_plays example
+# skvidal@fedoraproject.org
+# rbean@redhat.com
+# karsten@redhat.com  changes for fedora-messaging
+
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pwd
+import logging
+
+from fedora_messaging.api import Message, publish
+from fedora_messaging.exceptions import PublishReturned, ConnectionException
+
+try:
+    from ansible.plugins.callback import CallbackBase
+except ImportError:
+    # Ansible v1 compat
+    CallbackBase = object
+
+LOGGER = logging.getLogger(__name__)
+
+def getlogin():
+    try:
+        user = os.getlogin()
+    except OSError as e:
+        user = pwd.getpwuid(os.geteuid())[0]
+    return user
+
+
+class CallbackModule(CallbackBase):
+    """ Publish playbook starts and stops to fedora-messaging. """
+
+    playbook_path = None
+
+    def __init__(self):
+        pass
+
+    def playbook_on_play_start(self, pattern):
+        # This gets called once for each play.. but we just issue a message once
+        # for the first one.  One per "playbook"
+        play = getattr(self, "play", None)
+        if play:
+            # figure out where the playbook FILE is
+            path = os.path.abspath(play.playbook.filename)
+
+            # Bail out early without publishing if we're in --check mode
+            if play.playbook.check:
+                return
+
+            if not self.playbook_path:
+                try:
+                    msg = Message(
+                        topic="ansible.playbook.start",
+                        body={
+                            'playbook': path,
+                            'userid': getlogin(),
+                            'extra_vars': play.playbook.extra_vars,
+                            'inventory': play.playbook.inventory.host_list,
+                            'playbook_checksum': play.playbook.check,
+                            'check': play.playbook.check
+                        }
+                    )
+                    publish(msg)
+                except PublishReturned as e:
+                    LOGGER.warning(
+                        "Fedora Messaging broker rejected message %s: %s", msg.id, e
+                    )
+                except ConnectionException as e:
+                    LOGGER.warning("Error sending message %s: %s", msg.id, e)
+                self.playbook_path = path
+
+    def playbook_on_stats(self, stats):
+        if not self.playbook_path:
+            return
+
+        results = dict([(h, stats.summarize(h)) for h in stats.processed])
+        try:
+            msg = Message(
+                topic="ansible.playbook.complete",
+                body={
+                    'playbook': self.playbook_path,
+                    'userid': getlogin(),
+                    'results': results
+                )
+            )
+            publish(msg)
+        except PublishReturned as e:
+            LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e)
+        except ConnectionException as e:
+            LOGGER.warning("Error sending message %s: %s", msg.id, e)

callback_plugins/fedora_messaging_callback2.py

@@ -0,0 +1,121 @@
+# (C) 2012, Michael DeHaan, <michael.dehaan@gmail.com>
+# based on the log_plays example
+# skvidal@fedoraproject.org
+# rbean@redhat.com
+# karsten@redhat.com  changes for fedora-messaging
+
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pwd
+import logging
+
+from fedora_messaging.api import Message, publish
+from fedora_messaging.exceptions import PublishReturned, ConnectionException
+
+try:
+    from ansible.plugins.callback import CallbackBase
+except ImportError:
+    # Ansible v1 compat
+    CallbackBase = object
+
+try:
+    from ansible.utils.hashing import secure_hash
+except ImportError:
+    from ansible.utils import md5 as secure_hash
+
+LOGGER = logging.getLogger(__name__)
+
+def getlogin():
+    try:
+        user = os.getlogin()
+    except OSError as e:
+        user = pwd.getpwuid(os.geteuid())[0]
+    return user
+
+
+class CallbackModule(CallbackBase):
+    """ Publish playbook starts and stops to fedora_messaging. """
+
+    CALLBACK_NAME = "fedora_messaging_callback2"
+    CALLBACK_TYPE = "notification"
+    CALLBACK_VERSION = 2.0
+    CALLBACK_NEEDS_WHITELIST = True
+
+    playbook_path = None
+
+    def __init__(self):
+        self.play = None
+        self.playbook = None
+
+        super(CallbackModule, self).__init__()
+
+    def set_play_context(self, play_context):
+        self.play_context = play_context
+
+    def v2_playbook_on_start(self, playbook):
+        self.playbook = playbook
+
+    def v2_playbook_on_play_start(self, play):
+        # This gets called once for each play.. but we just issue a message once
+        # for the first one.  One per "playbook"
+        if self.playbook:
+            # figure out where the playbook FILE is
+            path = os.path.abspath(self.playbook._file_name)
+
+            # Bail out early without publishing if we're in --check mode
+            if self.play_context.check_mode:
+                return
+
+            if not self.playbook_path:
+                try:
+                    msg = Message(
+                        topic="ansible.playbook.start",
+                        body={
+                            'playbook': path,
+                            'userid': getlogin(),
+                            'extra_vars': play._variable_manager.extra_vars,
+                            'inventory': play._variable_manager._inventory._sources,
+                            'playbook_checksum': secure_hash(path),
+                            'check': self.play_context.check_mode
+                        }
+                    )
+                    publish(msg)
+                except PublishReturned as e:
+                    LOGGER.warning(
+                        "Fedora Messaging broker rejected message %s: %s", msg.id, e
+                    )
+                except ConnectionException as e:
+                    LOGGER.warning("Error sending message %s: %s", msg.id, e)
+                self.playbook_path = path
+
+    def v2_playbook_on_stats(self, stats):
+        if not self.playbook_path:
+            return
+
+        results = dict([(h, stats.summarize(h)) for h in stats.processed])
+        try:
+            msg = Message(
+                topic="ansible.playbook.complete",
+                body={
+                    'playbook': self.playbook_path,
+                    'userid': getlogin(),
+                    'results': results
+                }
+            )
+            publish(msg)
+        except PublishReturned as e:
+            LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e)
+        except ConnectionException as e:
+            LOGGER.warning("Error sending message %s: %s", msg.id, e)

playbooks/groups/datagrepper.yml

@@ -19,7 +19,10 @@
   - hosts
   - fas_client
   - collectd/base
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  - { role: rabbit/user,
+      username: "datagrepper{{ env_suffix }}"}
   - rsyncd
   - sudo
   - { role: openvpn/client,

playbooks/groups/github2fedmsg.yml

@@ -53,4 +53,5 @@
 
   roles:
   - github2fedmsg
-  - fedmsg/base
+  - { role: fedmsg/base, when: deployment_type == "prod" }
+  - { role: rabbit/user, when: deployment_type == "stg" }

playbooks/groups/mailman.yml

@@ -98,7 +98,11 @@
     mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}"
     mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}"
     mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}"
-  - fedmsg/base
+  -  { role: fedmsg/base,
+       when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "mailman{{ env_suffix }}"}
 
   tasks:
   - name: install more needed packages

playbooks/groups/mirrormanager.yml

@@ -100,7 +100,11 @@
   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
 
   roles:
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "mirrormanager{{ env_suffix }}"}
 
   handlers:
   - import_tasks: "{{ handlers_path }}/restart_services.yml"

playbooks/groups/noc.yml

@@ -64,7 +64,8 @@
   - { role: dhcp_server, when: datacenter == 'phx2' }
   - { role: tftp_server, when: datacenter == 'phx2' }
   - nagios_server
-  - fedmsg/base
+  - { role: fedmsg/base, when: deployment_type == "prod" }
+  - { role: rabbit/user, when: deployment_type == "stg" }
 
   tasks:
   - name: install some packages which arent in playbooks

playbooks/groups/notifs-backend.yml

@@ -25,7 +25,11 @@
   - fas_client
   - nagios_client
   - collectd/base
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "notifs-backend{{ env_suffix }}"}
   - sudo
   # The proxies don't actually need to talk to these hosts so we won't bother
   # putting them on the vpn.

playbooks/groups/notifs-web.yml

@@ -23,7 +23,11 @@
   - fas_client
   - collectd/base
   - mod_wsgi
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "notifs-web{{ env_suffix }}"}
   - notifs/frontend
   - sudo
   - { role: openvpn/client,

playbooks/groups/pdc.yml

@@ -44,7 +44,11 @@
   - role: openvpn/client
     when: env != "staging"
   - mod_wsgi
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "pdc{{ env_suffix }}"}
   - pdc/frontend
 
 - name: stuff just for the backend nodes

playbooks/groups/sundries.yml

@@ -51,6 +51,15 @@
     when: master_sundries_node|bool
   - role: developer/build
     when: master_sundries_node|bool
+  - { role: fedmsg/base,
+      when:
+        - master_sundries_node|bool
+        - deployment_type == "prod" }
+  - { role: rabbit/user,
+      username: "sundries{{ env_suffix }}",
+      when:
+        - master_sundries_node|bool
+        - deployment_type == "stg" }
   - role: fedmsg/base
     when: master_sundries_node|bool
   - role: nfs/client

playbooks/groups/value.yml

@@ -18,7 +18,11 @@
   - fas_client
   - collectd/base
   - apache
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "value{{ env_suffix }}"}
   - fedmsg/irc
   - supybot
   - sudo

playbooks/groups/wiki.yml

@@ -26,7 +26,13 @@
   - fas_client
   - collectd/base
   - apache
-  - fedmsg/base
+  - { fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "wiki{{ env_suffix }}"}
+  - role: rabbit/queue
+    username: "wiki{{ env_suffix }}"
   - { role: nfs/client, when: env == "staging", mnt_dir: '/mnt/web/attachments',  nfs_src_dir: 'fedora_app_staging/app/attachments' }
   - { role: nfs/client, when: env != "staging", mnt_dir: '/mnt/web/attachments',  nfs_src_dir: 'fedora_app/app/attachments' }
   - mediawiki

playbooks/groups/zanata2fedmsg.yml

@@ -53,4 +53,8 @@
 
   roles:
   - zanata2fedmsg
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "zanata{{ env_suffix }}"}

playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml

@@ -34,7 +34,11 @@
 
   roles:
   - basessh
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "happipstgfedorainfracloud{{ env_suffix }}"}
   - { role: letsencrypt, site_name: 'happinesspackets-stg.fedorainfracloud.org' }
 
   handlers:

playbooks/hosts/happinesspackets.fedorainfracloud.org.yml

@@ -34,7 +34,11 @@
 
   roles:
   - basessh
-  - fedmsg/base
+  - { role: fedmsg/base,
+      when: deployment_type == "prod" }
+  # Set up for fedora-messaging
+  - { role: rabbit/user,
+      username: "happipfedorainfracloud{{ env_suffix }}"}
   - { role: letsencrypt, site_name: 'happinesspackets.fedorainfracloud.org' }
 
   handlers:

roles/datagrepper/tasks/main.yml

@@ -3,6 +3,7 @@
   with_items:
   - datagrepper
   - python-psycopg2
+  - fedora-messaging
   tags:
   - packages
   - datagrepper
@@ -71,3 +72,35 @@
 
 # selinux policy has been intentionally omitted since that is obtained from fedmsg/base
 
+- name: Create /etc/pki/fedora-messaging
+  file:
+    dest: /etc/pki/fedora-messaging
+    mode: 0775
+    owner: root
+    group: root
+    state: directory
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging cert
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt"
+    dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem
+    mode: 0644
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora datagrepper fedora-messaging key
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key"
+    dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem
+    mode: 0640
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config

roles/mailman/tasks/main.yml

@@ -115,6 +115,7 @@
   - python34-PyYAML
   # mailman soft dep to convert html to plaintext
   - lynx
+  - fedora-messaging
   tags:
   - packages
   - mailman
@@ -554,3 +555,37 @@
   - webui-warm-up-cache
   tags: mailman
   when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev')
+
+- name: Create /etc/pki/fedora-messaging
+  file:
+    dest: /etc/pki/fedora-messaging
+    mode: 0775
+    owner: root
+    group: root
+    state: directory
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+# FIXME: Need to create a mailman cert
+- name: Deploy the Fedora mailman fedora-messaging cert
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mailman{{env_suffix}}.crt"
+    dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-cert.pem
+    mode: 0644
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora infra fedora-messaging key
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mailman{{env_suffix}}.key"
+    dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-key.pem
+    mode: 0640
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config

roles/mediawiki/tasks/main.yml

@@ -72,6 +72,47 @@
   - config
   - mediawiki
 
+#- name: adding fedora-messaging emit
+#  copy: src=fedora-message-emit.php dest=/usr/share/{{ wikiver }}/extensions/fedora-messaging-emit.php owner=root group=root mode=775
+#  tags:
+#  - config
+#  - mediawiki
+
+- name: Create /etc/pki/fedora-messaging
+  file:
+    dest: /etc/pki/fedora-messaging
+    mode: 0775
+    owner: root
+    group: root
+    state: directory
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+# FIXME: We currently don't seem to have a wiki cert, need to create one
+- name: Deploy the Fedora wiki fedora-messaging cert
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mediawiki{{env_suffix}}.crt"
+    dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-cert.pem
+    mode: 0644
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+# FIXME: We currently don't seem to have a wiki key, need to create one
+- name: Deploy the Fedora wiki fedora-messaging key
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mediawiki{{env_suffix}}.key"
+    dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-key.pem
+    mode: 0640
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
 - name: startup apache
   service: name=httpd enabled=yes state=started
   tags:

roles/mediawiki/templates/LocalSettings.php.fp.j2

@@ -303,6 +303,7 @@ $wgNamespacesToBeSearchedDefault = array(
         NS_TEST_RESULTS_TALK    => false
 );
 require_once "$IP/extensions/fedmsg-emit.php";
+# require_once "$IP/extensions/fedora-messaging-emit.php";
 require_once "$IP/extensions/HTTP302Found/HTTP302Found.php";
 require_once "$IP/extensions/RSS/RSS.php";
 require_once "$IP/extensions/FedoraDocsRedirect/FedoraDocsRedirect.php";

roles/mirrormanager/backend/tasks/main.yml

@@ -9,6 +9,7 @@
   - bzip2
   - python-psycopg2
   - fedmsg
+  - fedora-messaging
   - jq
   - geolite2-city
   - geolite2-country
@@ -86,6 +87,40 @@
   - config
   when: env != 'staging'
 
+- name: Create /etc/pki/fedora-messaging
+  file:
+    dest: /etc/pki/fedora-messaging
+    mode: 0775
+    owner: root
+    group: root
+    state: directory
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+# FIXME: do we need to create a mirrormanager cert ?
+- name: Deploy the Fedora mirrormanager fedora-messaging cert
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mirrormanager{{env_suffix}}.crt"
+    dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-cert.pem
+    mode: 0644
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
+- name: Deploy the Fedora infra fedora-messaging key
+  copy:
+    src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mirrormanager{{env_suffix}}.key"
+    dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-key.pem
+    mode: 0640
+    owner: root
+    group: root
+  when: "deployment_type is defined"
+  tags:
+  - config
+
 # To decrease the crawl duration on the mirrors we have been
 # recommending to lower the default value of vfs_cache_pressure
 # from 100 to 10. This causes the kernel to prefer to keep dentries