diff --git a/playbooks/groups/pagure.yml b/playbooks/groups/pagure.yml index e06dc4894b..305e3db68b 100644 --- a/playbooks/groups/pagure.yml +++ b/playbooks/groups/pagure.yml @@ -41,23 +41,8 @@ - "/srv/private/ansible/vars.yml" - "{{ vars_path }}/{{ ansible_distribution }}.yml" - pre_tasks: - - name: install fedmsg-relay - package: name=fedmsg-relay state=present - when: env != 'pagure-staging' - tags: - - pagure - - pagure/fedmsg - - name: and start it - service: name=fedmsg-relay state=started - when: env != 'pagure-staging' - tags: - - pagure - - pagure/fedmsg - roles: - pagure/frontend - - {role: pagure/fedmsg, when: env != 'pagure-staging'} handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml" diff --git a/roles/pagure/fedmsg/files/selinux/fedmsg.mod b/roles/pagure/fedmsg/files/selinux/fedmsg.mod deleted file mode 100644 index 25e47ae694..0000000000 Binary files a/roles/pagure/fedmsg/files/selinux/fedmsg.mod and /dev/null differ diff --git a/roles/pagure/fedmsg/files/selinux/fedmsg.pp b/roles/pagure/fedmsg/files/selinux/fedmsg.pp deleted file mode 100644 index 17a25943f7..0000000000 Binary files a/roles/pagure/fedmsg/files/selinux/fedmsg.pp and /dev/null differ diff --git a/roles/pagure/fedmsg/files/selinux/fedmsg.te b/roles/pagure/fedmsg/files/selinux/fedmsg.te deleted file mode 100644 index 6ce38d40f6..0000000000 --- a/roles/pagure/fedmsg/files/selinux/fedmsg.te +++ /dev/null @@ -1,21 +0,0 @@ - -module fedmsg 1.1; - -require { - type anon_inodefs_t; - type httpd_t; - class file write; -} - -require { - type ptmx_t; - type httpd_t; - class chr_file getattr; -} - -#============= httpd_t ============== -# For basic port binding -allow httpd_t anon_inodefs_t:file write; -# So that psutil can work from /etc/fedmsg.d/logging.py -allow httpd_t ptmx_t:chr_file getattr; - diff --git a/roles/pagure/fedmsg/tasks/main.yml b/roles/pagure/fedmsg/tasks/main.yml deleted file mode 100644 index 90ca99fd1a..0000000000 --- a/roles/pagure/fedmsg/tasks/main.yml +++ /dev/null @@ -1,162 +0,0 @@ ---- -# tasklist for setting up fedmsg -# This is the base set of files needed for fedmsg - -- name: install needed packages - package: name={{ item }} state=present - with_items: - - fedmsg - - libsemanage-python - - python-psutil - tags: - - packages - - pagure - - pagure/fedmsg - when: ansible_distribution_major_version|int < 8 and ansible_distribution == 'RedHat' - -- name: install needed packages - package: name={{ item }} state=present - with_items: - - fedmsg - - python3-libsemanage - - python3-psutil - tags: - - packages - - pagure - - pagure/fedmsg - when: ansible_distribution_major_version|int >= 8 and ansible_distribution == 'RedHat' - - -# We use setgid here so that the monitoring sockets created by fedmsg services -# are accessible to the nrpe group. -- name: create a /var/run/fedmsg dir with setgid for monitoring. - file: > - dest=/var/run/fedmsg - mode=2775 - owner=fedmsg - group=nrpe - state=directory - tags: - - pagure - - pagure - - pagure/fedmsg - -- name: setup /etc/fedmsg.d directory - file: path=/etc/fedmsg.d owner=root group=root mode=0755 state=directory - tags: - - pagure - - pagure/fedmsg - - config - -# Any files that change need to restart any services that depend on them. A -# trick here is that some hosts have an httpd that uses fedmsg, while others do -# not. Some hosts have a fedmsg-hub that uses this config, while others do not. -# Our handlers in handlers/restart_services.yml are smart enough to -# *conditionally* restart these services, only if they are installed on the -# system. -- name: setup basic /etc/fedmsg.d/ contents - template: > - src="{{ item }}.j2" - dest="/etc/fedmsg.d/{{ item }}" - owner=root - group=root - mode=644 - with_items: - - ssl.py - - relay.py - - logging.py - - base.py - tags: - - config - - fedmsgdconfig - - pagure - - pagure/fedmsg - notify: - - reload httpd - - restart fedmsg-relay - -- name: Remove unwanted files - file: dest=/etc/fedmsg.d/{{item}} state=absent - with_items: - - endpoints.py - tags: - - config - - fedmsgdconfig - - pagure - - pagure/fedmsg - notify: - - reload httpd - - restart fedmsg-relay - -- name: setup /etc/pki/fedmsg directory - file: path=/etc/pki/fedmsg owner=root group=root mode=0755 state=directory - tags: - - config - - pagure - - pagure/fedmsg - -- name: install fedmsg ca.cert - copy: > - src="{{ private }}/files/fedmsg-certs/keys/ca.crt" - dest=/etc/pki/fedmsg/ca.crt - owner=root - group=root - mode=0644 - tags: - - config - - pagure - - pagure/fedmsg - -- name: fedmsg certs - copy: > - src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(inventory_hostname)}}.crt" - dest=/etc/pki/fedmsg/ - mode=644 - owner={{item['owner']}} - group={{item['group']}} - with_items: - - "{{ fedmsg_certs }}" - when: fedmsg_certs != [] - tags: - - config - - pagure - - pagure/fedmsg - -- name: fedmsg keys - copy: > - src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(inventory_hostname)}}.key" - dest=/etc/pki/fedmsg/ - mode=0640 - owner={{item['owner']}} - group={{item['group']}} - with_items: - - "{{ fedmsg_certs }}" - when: fedmsg_certs != [] - tags: - - config - - pagure - - pagure/fedmsg - -# Three tasks for handling our custom selinux module -- name: ensure a directory exists for our custom selinux module - file: dest=/usr/local/share/fedmsg state=directory - tags: - - selinux - - pagure - - pagure/fedmsg - -- name: copy over our custom selinux module - copy: src=selinux/fedmsg.pp dest=/usr/local/share/fedmsg/fedmsg.pp - register: selinux_module - tags: - - selinux - - pagure - - pagure/fedmsg - -- name: install our custom selinux module - command: semodule -i /usr/local/share/fedmsg/fedmsg.pp - when: selinux_module is changed - tags: - - selinux - - pagure - - pagure/fedmsg diff --git a/roles/pagure/fedmsg/templates/base.py.j2 b/roles/pagure/fedmsg/templates/base.py.j2 deleted file mode 100644 index 31a8e93da1..0000000000 --- a/roles/pagure/fedmsg/templates/base.py.j2 +++ /dev/null @@ -1,68 +0,0 @@ -config = dict( - - # Tell every call to `fedmsg.publish` to use the relay - active=True, - cert_prefix="pagure", - - topic_prefix="{{ fedmsg_prefix }}", - environment="{{ fedmsg_env }}", - - # This used to be set to 1 for safety, but it turns out it was - # excessive. It is the number of seconds that fedmsg should sleep - # after it has initialized, but before it begins to try and send any - # messages. If set to a non-zero value, this will slow down one-off - # fedmsg scripts like the git post-receive hook and pkgdb2branch. - # If we are experiencing message-loss problems, one of the first things - # to try should be to turn this number up to a non-zero value. '1' should - # be more than sufficient. - post_init_sleep=0.4, - - # This is the number of milliseconds to wait before timing out on - # connections.. notably to the fedmsg-relay in the event that it has - # crashed. - zmq_linger=2000, - - # Default is 0 - high_water_mark=0, - io_threads=1, - - # We almost always want the fedmsg-hub to be sending messages with zmq as - # opposed to amqp or stomp. The only exception will be the bugzilla - # amqp<->zmq bridge service. - zmq_enabled=True, - - # When subscribing to messages, we want to allow splats ('*') so we tell the - # hub to not be strict when comparing messages topics to subscription - # topics. - zmq_strict=False, - - # See the following - # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html - # - http://api.zeromq.org/3-2:zmq-setsockopt - zmq_tcp_keepalive=1, - zmq_tcp_keepalive_cnt=3, - zmq_tcp_keepalive_idle=60, - zmq_tcp_keepalive_intvl=5, -) - -# This option adds an IPC socket by which we can monitor hub health. -try: - import os - import psutil - - pid = os.getpid() - proc = [p for p in psutil.process_iter() if p.pid == pid][0] - - # proc.name is a method on modern versions of psutil. - name = proc.name - if callable(name): - name = name() - - config['moksha.monitoring.socket'] = \ - 'ipc:///var/run/fedmsg/monitoring-%s.socket' % name - config['moksha.monitoring.socket.mode'] = '770' -except (OSError, ImportError): - # We run into issues when trying to import psutil from mod_wsgi on rhel7 - # but this feature is of no concern in that context, so just fail quietly. - # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837 - pass diff --git a/roles/pagure/fedmsg/templates/logging.py.j2 b/roles/pagure/fedmsg/templates/logging.py.j2 deleted file mode 100644 index 9dcbcc8ba8..0000000000 --- a/roles/pagure/fedmsg/templates/logging.py.j2 +++ /dev/null @@ -1,164 +0,0 @@ -# Setup fedmsg logging. - -# All of these modules are just used by the ContextInjector below. -import inspect -import logging -import os -import socket -import traceback - -psutil = None -try: - import psutil -except (OSError, ImportError): - # We run into issues when trying to import psutil from inside mod_wsgi on - # rhel7. If we hit that here, then just fail quietly. - # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837 - pass - - -class ContextInjector(logging.Filter): - """ Logging filter that adds context to log records. - - Filters are typically used to "filter" log records. They declare a filter - method that can return True or False. Only records with 'True' will - actually be logged. - - Here, we somewhat abuse the concept of a filter. We always return true, - but we use the opportunity to hang important contextual information on the - log record to later be used by the logging Formatter. We don't normally - want to see all this stuff in normal log records, but we *do* want to see - it when we are emailed error messages. Seeing an error, but not knowing - which host it comes from, is not that useful. - - http://docs.python.org/2/howto/logging-cookbook.html#filters-contextual - """ - - def filter(self, record): - current_process = ContextInjector.get_current_process() - current_hostname = socket.gethostname() - - record.host = current_hostname - if "notifs-backend" in current_hostname: - return False - record.proc = current_process - record.pid = current_process.pid - record.proc_name = current_process.name - record.command_line = current_process.cmdline - # These are callabls on more modern versions of psutil. - if callable(record.proc_name): - record.proc_name = record.proc_name() - if callable(record.command_line): - record.command_line = record.command_line() - record.command_line = " ".join(record.command_line) - record.callstack = self.format_callstack() - return True - - @staticmethod - def format_callstack(): - for i, frame in enumerate(f[0] for f in inspect.stack()): - if not '__name__' in frame.f_globals: - continue - modname = frame.f_globals['__name__'].split('.')[0] - if modname != "logging": - break - - def _format_frame(frame): - return ' File "%s", line %i in %s\n %s' % (frame) - - stack = traceback.extract_stack() - stack = stack[:-i] - return "\n".join([_format_frame(frame) for frame in stack]) - - @staticmethod - def get_current_process(): - mypid = os.getpid() - - if not psutil: - raise OSError("Could not import psutil for %r" % mypid) - - for proc in psutil.process_iter(): - if proc.pid == mypid: - return proc - - # This should be impossible. - raise ValueError("Could not find process %r" % mypid) - - @classmethod - def __json__(cls): - """ We need to be jsonifiable for "fedmsg-config" """ - return {'name': 'ContextInjector'} - - -hefty_format = """Message -------- -[%(asctime)s][%(name)10s %(levelname)7s] -%(message)s - -Process Details ---------------- -host: %(host)s -PID: %(pid)s -name: %(proc_name)s -command: %(command_line)s - -Callstack that lead to the logging statement --------------------------------------------- -%(callstack)s -""" - - -# See the following for constraints on this format http://bit.ly/Xn1WDn -config = dict( - logging=dict( - version=1, - formatters=dict( - bare={ - "datefmt": "%Y-%m-%d %H:%M:%S", - "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" - }, - hefty={ - "datefmt": "%Y-%m-%d %H:%M:%S", - "format": hefty_format, - }, - ), - filters=dict( - context={ - # This "()" syntax in the stdlib doesn't seem to be documented - # anywhere. I had to read - # /usr/lib64/python2.7/logging/config.py to figure it out. - "()": ContextInjector, - }, - ), - handlers=dict( - console={ - "class": "logging.StreamHandler", - "formatter": "bare", - "level": "INFO", - "stream": "ext://sys.stdout", - }, - mailer={ - "class": "logging.handlers.SMTPHandler", - "formatter": "hefty", - "filters": ["context"], - "level": "ERROR", - "mailhost": "bastion.vpn.fedoraproject.org", - "fromaddr": "fedmsg@fedoraproject.org", - "toaddrs": ["sysadmin-datanommer-members@fedoraproject.org"], - "subject": "fedmsg error log (pagure)", - }, - ), - loggers=dict( - fedmsg={ - "level": "INFO", - "propagate": False, - "handlers": ["console", "mailer"], - }, - moksha={ - "level": "INFO", - "propagate": False, - "handlers": ["console", "mailer"], - }, - ), - ), -) diff --git a/roles/pagure/fedmsg/templates/relay.py.j2 b/roles/pagure/fedmsg/templates/relay.py.j2 deleted file mode 100644 index cd620d927b..0000000000 --- a/roles/pagure/fedmsg/templates/relay.py.j2 +++ /dev/null @@ -1,16 +0,0 @@ -config = dict( - endpoints={ - # This is the output side of the relay to which all other - # services can listen. - "relay_outbound": [ - # Messages emerge here - #"tcp://pagure.io:9940", - "tcp://{{inventory_hostname}}:9940", - ], - }, - - # wsgi scripts on the frontend talk back here - relay_inbound=[ - "tcp://{{inventory_hostname}}:9941", - ], -) diff --git a/roles/pagure/fedmsg/templates/ssl.py.j2 b/roles/pagure/fedmsg/templates/ssl.py.j2 deleted file mode 100644 index 4db87f149c..0000000000 --- a/roles/pagure/fedmsg/templates/ssl.py.j2 +++ /dev/null @@ -1,16 +0,0 @@ - -config = dict( - sign_messages=True, - validate_signatures=True, - ssldir="/etc/pki/fedmsg", - - crl_location="https://fedoraproject.org/fedmsg/crl.pem", - crl_cache="/var/run/fedmsg/crl.pem", - crl_cache_expiry=86400, # Daily - - certnames=dict([ - ("shell.{{inventory_hostname_short}}", "shell-{{inventory_hostname}}"), - ("pagure.{{inventory_hostname_short}}", "pagure-{{inventory_hostname}}"), - ]), -) -