diff --git a/callback_plugins/fedora_messaging_callback.py b/callback_plugins/fedora_messaging_callback.py new file mode 100644 index 0000000000..620fe456eb --- /dev/null +++ b/callback_plugins/fedora_messaging_callback.py @@ -0,0 +1,103 @@ +# (C) 2012, Michael DeHaan, +# based on the log_plays example +# skvidal@fedoraproject.org +# rbean@redhat.com +# karsten@redhat.com changes for fedora-messaging + +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import pwd +import logging + +from fedora_messaging.api import Message, publish +from fedora_messaging.exceptions import PublishReturned, ConnectionException + +try: + from ansible.plugins.callback import CallbackBase +except ImportError: + # Ansible v1 compat + CallbackBase = object + +LOGGER = logging.getLogger(__name__) + +def getlogin(): + try: + user = os.getlogin() + except OSError as e: + user = pwd.getpwuid(os.geteuid())[0] + return user + + +class CallbackModule(CallbackBase): + """ Publish playbook starts and stops to fedora-messaging. """ + + playbook_path = None + + def __init__(self): + pass + + def playbook_on_play_start(self, pattern): + # This gets called once for each play.. but we just issue a message once + # for the first one. One per "playbook" + play = getattr(self, "play", None) + if play: + # figure out where the playbook FILE is + path = os.path.abspath(play.playbook.filename) + + # Bail out early without publishing if we're in --check mode + if play.playbook.check: + return + + if not self.playbook_path: + try: + msg = Message( + topic="ansible.playbook.start", + body={ + 'playbook': path, + 'userid': getlogin(), + 'extra_vars': play.playbook.extra_vars, + 'inventory': play.playbook.inventory.host_list, + 'playbook_checksum': play.playbook.check, + 'check': play.playbook.check + } + ) + publish(msg) + except PublishReturned as e: + LOGGER.warning( + "Fedora Messaging broker rejected message %s: %s", msg.id, e + ) + except ConnectionException as e: + LOGGER.warning("Error sending message %s: %s", msg.id, e) + self.playbook_path = path + + def playbook_on_stats(self, stats): + if not self.playbook_path: + return + + results = dict([(h, stats.summarize(h)) for h in stats.processed]) + try: + msg = Message( + topic="ansible.playbook.complete", + body={ + 'playbook': self.playbook_path, + 'userid': getlogin(), + 'results': results + ) + ) + publish(msg) + except PublishReturned as e: + LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e) + except ConnectionException as e: + LOGGER.warning("Error sending message %s: %s", msg.id, e) diff --git a/callback_plugins/fedora_messaging_callback2.py b/callback_plugins/fedora_messaging_callback2.py new file mode 100644 index 0000000000..dc4176376c --- /dev/null +++ b/callback_plugins/fedora_messaging_callback2.py @@ -0,0 +1,121 @@ +# (C) 2012, Michael DeHaan, +# based on the log_plays example +# skvidal@fedoraproject.org +# rbean@redhat.com +# karsten@redhat.com changes for fedora-messaging + +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +import os +import pwd +import logging + +from fedora_messaging.api import Message, publish +from fedora_messaging.exceptions import PublishReturned, ConnectionException + +try: + from ansible.plugins.callback import CallbackBase +except ImportError: + # Ansible v1 compat + CallbackBase = object + +try: + from ansible.utils.hashing import secure_hash +except ImportError: + from ansible.utils import md5 as secure_hash + +LOGGER = logging.getLogger(__name__) + +def getlogin(): + try: + user = os.getlogin() + except OSError as e: + user = pwd.getpwuid(os.geteuid())[0] + return user + + +class CallbackModule(CallbackBase): + """ Publish playbook starts and stops to fedora_messaging. """ + + CALLBACK_NAME = "fedora_messaging_callback2" + CALLBACK_TYPE = "notification" + CALLBACK_VERSION = 2.0 + CALLBACK_NEEDS_WHITELIST = True + + playbook_path = None + + def __init__(self): + self.play = None + self.playbook = None + + super(CallbackModule, self).__init__() + + def set_play_context(self, play_context): + self.play_context = play_context + + def v2_playbook_on_start(self, playbook): + self.playbook = playbook + + def v2_playbook_on_play_start(self, play): + # This gets called once for each play.. but we just issue a message once + # for the first one. One per "playbook" + if self.playbook: + # figure out where the playbook FILE is + path = os.path.abspath(self.playbook._file_name) + + # Bail out early without publishing if we're in --check mode + if self.play_context.check_mode: + return + + if not self.playbook_path: + try: + msg = Message( + topic="ansible.playbook.start", + body={ + 'playbook': path, + 'userid': getlogin(), + 'extra_vars': play._variable_manager.extra_vars, + 'inventory': play._variable_manager._inventory._sources, + 'playbook_checksum': secure_hash(path), + 'check': self.play_context.check_mode + } + ) + publish(msg) + except PublishReturned as e: + LOGGER.warning( + "Fedora Messaging broker rejected message %s: %s", msg.id, e + ) + except ConnectionException as e: + LOGGER.warning("Error sending message %s: %s", msg.id, e) + self.playbook_path = path + + def v2_playbook_on_stats(self, stats): + if not self.playbook_path: + return + + results = dict([(h, stats.summarize(h)) for h in stats.processed]) + try: + msg = Message( + topic="ansible.playbook.complete", + body={ + 'playbook': self.playbook_path, + 'userid': getlogin(), + 'results': results + } + ) + publish(msg) + except PublishReturned as e: + LOGGER.warning("Fedora Messaging broker rejected message %s: %s", msg.id, e) + except ConnectionException as e: + LOGGER.warning("Error sending message %s: %s", msg.id, e) diff --git a/playbooks/groups/datagrepper.yml b/playbooks/groups/datagrepper.yml index 3d87af188e..f2e2e31e1a 100644 --- a/playbooks/groups/datagrepper.yml +++ b/playbooks/groups/datagrepper.yml @@ -19,7 +19,10 @@ - hosts - fas_client - collectd/base - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + - { role: rabbit/user, + username: "datagrepper{{ env_suffix }}"} - rsyncd - sudo - { role: openvpn/client, diff --git a/playbooks/groups/github2fedmsg.yml b/playbooks/groups/github2fedmsg.yml index 4c208c824a..129a58bf5f 100644 --- a/playbooks/groups/github2fedmsg.yml +++ b/playbooks/groups/github2fedmsg.yml @@ -53,4 +53,5 @@ roles: - github2fedmsg - - fedmsg/base + - { role: fedmsg/base, when: deployment_type == "prod" } + - { role: rabbit/user, when: deployment_type == "stg" } diff --git a/playbooks/groups/mailman.yml b/playbooks/groups/mailman.yml index 7bfce6a85d..df3e834591 100644 --- a/playbooks/groups/mailman.yml +++ b/playbooks/groups/mailman.yml @@ -98,7 +98,11 @@ mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}" mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}" mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}" - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "mailman{{ env_suffix }}"} tasks: - name: install more needed packages diff --git a/playbooks/groups/mirrormanager.yml b/playbooks/groups/mirrormanager.yml index c5a7722f48..6e73a6216b 100644 --- a/playbooks/groups/mirrormanager.yml +++ b/playbooks/groups/mirrormanager.yml @@ -100,7 +100,11 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "mirrormanager{{ env_suffix }}"} handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/playbooks/groups/noc.yml b/playbooks/groups/noc.yml index f016091d43..db793bdb73 100644 --- a/playbooks/groups/noc.yml +++ b/playbooks/groups/noc.yml @@ -64,7 +64,8 @@ - { role: dhcp_server, when: datacenter == 'phx2' } - { role: tftp_server, when: datacenter == 'phx2' } - nagios_server - - fedmsg/base + - { role: fedmsg/base, when: deployment_type == "prod" } + - { role: rabbit/user, when: deployment_type == "stg" } tasks: - name: install some packages which arent in playbooks diff --git a/playbooks/groups/notifs-backend.yml b/playbooks/groups/notifs-backend.yml index 93df0c2f8b..b33c287445 100644 --- a/playbooks/groups/notifs-backend.yml +++ b/playbooks/groups/notifs-backend.yml @@ -25,7 +25,11 @@ - fas_client - nagios_client - collectd/base - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "notifs-backend{{ env_suffix }}"} - sudo # The proxies don't actually need to talk to these hosts so we won't bother # putting them on the vpn. diff --git a/playbooks/groups/notifs-web.yml b/playbooks/groups/notifs-web.yml index ec0e963a31..2a6ca86683 100644 --- a/playbooks/groups/notifs-web.yml +++ b/playbooks/groups/notifs-web.yml @@ -23,7 +23,11 @@ - fas_client - collectd/base - mod_wsgi - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "notifs-web{{ env_suffix }}"} - notifs/frontend - sudo - { role: openvpn/client, diff --git a/playbooks/groups/pdc.yml b/playbooks/groups/pdc.yml index b581507a6c..e7f7fea98b 100644 --- a/playbooks/groups/pdc.yml +++ b/playbooks/groups/pdc.yml @@ -44,7 +44,11 @@ - role: openvpn/client when: env != "staging" - mod_wsgi - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "pdc{{ env_suffix }}"} - pdc/frontend - name: stuff just for the backend nodes diff --git a/playbooks/groups/sundries.yml b/playbooks/groups/sundries.yml index 561dcaae61..88fbb781ca 100644 --- a/playbooks/groups/sundries.yml +++ b/playbooks/groups/sundries.yml @@ -51,6 +51,15 @@ when: master_sundries_node|bool - role: developer/build when: master_sundries_node|bool + - { role: fedmsg/base, + when: + - master_sundries_node|bool + - deployment_type == "prod" } + - { role: rabbit/user, + username: "sundries{{ env_suffix }}", + when: + - master_sundries_node|bool + - deployment_type == "stg" } - role: fedmsg/base when: master_sundries_node|bool - role: nfs/client diff --git a/playbooks/groups/value.yml b/playbooks/groups/value.yml index 8e2dd01df6..cef0765c6d 100644 --- a/playbooks/groups/value.yml +++ b/playbooks/groups/value.yml @@ -18,7 +18,11 @@ - fas_client - collectd/base - apache - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "value{{ env_suffix }}"} - fedmsg/irc - supybot - sudo diff --git a/playbooks/groups/wiki.yml b/playbooks/groups/wiki.yml index b3f4b7ecee..106e583dce 100644 --- a/playbooks/groups/wiki.yml +++ b/playbooks/groups/wiki.yml @@ -26,7 +26,13 @@ - fas_client - collectd/base - apache - - fedmsg/base + - { fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "wiki{{ env_suffix }}"} + - role: rabbit/queue + username: "wiki{{ env_suffix }}" - { role: nfs/client, when: env == "staging", mnt_dir: '/mnt/web/attachments', nfs_src_dir: 'fedora_app_staging/app/attachments' } - { role: nfs/client, when: env != "staging", mnt_dir: '/mnt/web/attachments', nfs_src_dir: 'fedora_app/app/attachments' } - mediawiki diff --git a/playbooks/groups/zanata2fedmsg.yml b/playbooks/groups/zanata2fedmsg.yml index 0694295efe..5885b0e2a7 100644 --- a/playbooks/groups/zanata2fedmsg.yml +++ b/playbooks/groups/zanata2fedmsg.yml @@ -53,4 +53,8 @@ roles: - zanata2fedmsg - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "zanata{{ env_suffix }}"} diff --git a/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml b/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml index f146c40f14..20e7c20439 100644 --- a/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml +++ b/playbooks/hosts/happinesspackets-stg.fedorainfracloud.org.yml @@ -34,7 +34,11 @@ roles: - basessh - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "happipstgfedorainfracloud{{ env_suffix }}"} - { role: letsencrypt, site_name: 'happinesspackets-stg.fedorainfracloud.org' } handlers: diff --git a/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml b/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml index 2cd1acd56c..a57e047e31 100644 --- a/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml +++ b/playbooks/hosts/happinesspackets.fedorainfracloud.org.yml @@ -34,7 +34,11 @@ roles: - basessh - - fedmsg/base + - { role: fedmsg/base, + when: deployment_type == "prod" } + # Set up for fedora-messaging + - { role: rabbit/user, + username: "happipfedorainfracloud{{ env_suffix }}"} - { role: letsencrypt, site_name: 'happinesspackets.fedorainfracloud.org' } handlers: diff --git a/roles/datagrepper/tasks/main.yml b/roles/datagrepper/tasks/main.yml index ccf35a3d01..e13497076d 100644 --- a/roles/datagrepper/tasks/main.yml +++ b/roles/datagrepper/tasks/main.yml @@ -3,6 +3,7 @@ with_items: - datagrepper - python-psycopg2 + - fedora-messaging tags: - packages - datagrepper @@ -71,3 +72,35 @@ # selinux policy has been intentionally omitted since that is obtained from fedmsg/base +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + when: "deployment_type is defined" + tags: + - config + +- name: Deploy the Fedora datagrepper fedora-messaging cert + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/datagrepper{{env_suffix}}.crt" + dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-cert.pem + mode: 0644 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + +- name: Deploy the Fedora datagrepper fedora-messaging key + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/datagrepper{{env_suffix}}.key" + dest: /etc/pki/fedora-messaging/datagrepper{{env_suffix}}-key.pem + mode: 0640 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml index 81fd7106a7..2109fea977 100644 --- a/roles/mailman/tasks/main.yml +++ b/roles/mailman/tasks/main.yml @@ -115,6 +115,7 @@ - python34-PyYAML # mailman soft dep to convert html to plaintext - lynx + - fedora-messaging tags: - packages - mailman @@ -554,3 +555,37 @@ - webui-warm-up-cache tags: mailman when: inventory_hostname.startswith('mailman01.phx2') or inventory_hostname.startswith('lists-dev') + +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + when: "deployment_type is defined" + tags: + - config + +# FIXME: Need to create a mailman cert +- name: Deploy the Fedora mailman fedora-messaging cert + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mailman{{env_suffix}}.crt" + dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-cert.pem + mode: 0644 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + +- name: Deploy the Fedora infra fedora-messaging key + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mailman{{env_suffix}}.key" + dest: /etc/pki/fedora-messaging/mailman{{env_suffix}}-key.pem + mode: 0640 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index 958782dd8e..843ae43587 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -72,6 +72,47 @@ - config - mediawiki +#- name: adding fedora-messaging emit +# copy: src=fedora-message-emit.php dest=/usr/share/{{ wikiver }}/extensions/fedora-messaging-emit.php owner=root group=root mode=775 +# tags: +# - config +# - mediawiki + +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + when: "deployment_type is defined" + tags: + - config + +# FIXME: We currently don't seem to have a wiki cert, need to create one +- name: Deploy the Fedora wiki fedora-messaging cert + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mediawiki{{env_suffix}}.crt" + dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-cert.pem + mode: 0644 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + +# FIXME: We currently don't seem to have a wiki key, need to create one +- name: Deploy the Fedora wiki fedora-messaging key + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mediawiki{{env_suffix}}.key" + dest: /etc/pki/fedora-messaging/mediawiki{{env_suffix}}-key.pem + mode: 0640 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + - name: startup apache service: name=httpd enabled=yes state=started tags: diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2 index ad39df9636..2ba00b8b61 100644 --- a/roles/mediawiki/templates/LocalSettings.php.fp.j2 +++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2 @@ -303,6 +303,7 @@ $wgNamespacesToBeSearchedDefault = array( NS_TEST_RESULTS_TALK => false ); require_once "$IP/extensions/fedmsg-emit.php"; +# require_once "$IP/extensions/fedora-messaging-emit.php"; require_once "$IP/extensions/HTTP302Found/HTTP302Found.php"; require_once "$IP/extensions/RSS/RSS.php"; require_once "$IP/extensions/FedoraDocsRedirect/FedoraDocsRedirect.php"; diff --git a/roles/mirrormanager/backend/tasks/main.yml b/roles/mirrormanager/backend/tasks/main.yml index 3793f6ffe5..20fff62ab0 100644 --- a/roles/mirrormanager/backend/tasks/main.yml +++ b/roles/mirrormanager/backend/tasks/main.yml @@ -9,6 +9,7 @@ - bzip2 - python-psycopg2 - fedmsg + - fedora-messaging - jq - geolite2-city - geolite2-country @@ -86,6 +87,40 @@ - config when: env != 'staging' +- name: Create /etc/pki/fedora-messaging + file: + dest: /etc/pki/fedora-messaging + mode: 0775 + owner: root + group: root + state: directory + when: "deployment_type is defined" + tags: + - config + +# FIXME: do we need to create a mirrormanager cert ? +- name: Deploy the Fedora mirrormanager fedora-messaging cert + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/issued/mirrormanager{{env_suffix}}.crt" + dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-cert.pem + mode: 0644 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + +- name: Deploy the Fedora infra fedora-messaging key + copy: + src: "{{ private }}/files/rabbitmq/{{env}}/pki/private/mirrormanager{{env_suffix}}.key" + dest: /etc/pki/fedora-messaging/mirrormanager{{env_suffix}}-key.pem + mode: 0640 + owner: root + group: root + when: "deployment_type is defined" + tags: + - config + # To decrease the crawl duration on the mirrors we have been # recommending to lower the default value of vfs_cache_pressure # from 100 to 10. This causes the kernel to prefer to keep dentries