diff --git a/playbooks/groups/anitya.yml b/playbooks/groups/anitya.yml index d66bdef5aa..bc595fc489 100644 --- a/playbooks/groups/anitya.yml +++ b/playbooks/groups/anitya.yml @@ -65,8 +65,15 @@ - include: "{{ tasks }}/apache.yml" roles: + - anitya/fedmsg - anitya/frontend + tasks: + - name: install fedmsg-relay + yum: pkg=fedmsg-relay state=installed + - name: and start it + service: name=fedmsg-relay state=started + tags: - anitya @@ -85,6 +92,7 @@ - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml roles: + - anitya/fedmsg - anitya/backend tags: @@ -92,21 +100,3 @@ handlers: - include: "{{ handlers }}/restart_services.yml" - -# This comes last because it requires usernames from some of the previous plays. -- name: finish up by setting up fedmsg - hosts: anitya - user: root - gather_facts: True - accelerate: "{{ accelerated }}" - - vars_files: - - /srv/web/infra/ansible/vars/global.yml - - "{{ private }}/vars.yml" - - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml - - roles: - - fedmsg/base - - handlers: - - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/anitya/fedmsg/files/selinux/fedmsg.mod b/roles/anitya/fedmsg/files/selinux/fedmsg.mod new file mode 100644 index 0000000000..25e47ae694 Binary files /dev/null and b/roles/anitya/fedmsg/files/selinux/fedmsg.mod differ diff --git a/roles/anitya/fedmsg/files/selinux/fedmsg.pp b/roles/anitya/fedmsg/files/selinux/fedmsg.pp new file mode 100644 index 0000000000..17a25943f7 Binary files /dev/null and b/roles/anitya/fedmsg/files/selinux/fedmsg.pp differ diff --git a/roles/anitya/fedmsg/files/selinux/fedmsg.te b/roles/anitya/fedmsg/files/selinux/fedmsg.te new file mode 100644 index 0000000000..6ce38d40f6 --- /dev/null +++ b/roles/anitya/fedmsg/files/selinux/fedmsg.te @@ -0,0 +1,21 @@ + +module fedmsg 1.1; + +require { + type anon_inodefs_t; + type httpd_t; + class file write; +} + +require { + type ptmx_t; + type httpd_t; + class chr_file getattr; +} + +#============= httpd_t ============== +# For basic port binding +allow httpd_t anon_inodefs_t:file write; +# So that psutil can work from /etc/fedmsg.d/logging.py +allow httpd_t ptmx_t:chr_file getattr; + diff --git a/roles/anitya/fedmsg/tasks/main.yml b/roles/anitya/fedmsg/tasks/main.yml new file mode 100644 index 0000000000..a8269ebecf --- /dev/null +++ b/roles/anitya/fedmsg/tasks/main.yml @@ -0,0 +1,118 @@ +--- +# tasklist for setting up fedmsg +# This is the base set of files needed for fedmsg + +- name: install needed packages + yum: pkg={{ item }} state=installed + with_items: + - fedmsg + - libsemanage-python + - python-psutil + - policycoreutils-python # This is in the kickstart now. Here for old hosts. + tags: + - packages + +# We use setgid here so that the monitoring sockets created by fedmsg services +# are accessible to the nrpe group. +- name: create a /var/run/fedmsg dir with setgid for monitoring. + file: > + dest=/var/run/fedmsg + mode=2775 + owner=fedmsg + group=nrpe + state=directory + +- name: setup /etc/fedmsg.d directory + file: path=/etc/fedmsg.d owner=root group=root mode=0755 state=directory + tags: + - config + +# Any files that change need to restart any services that depend on them. A +# trick here is that some hosts have an httpd that uses fedmsg, while others do +# not. Some hosts have a fedmsg-hub that uses this config, while others do not. +# Our handlers in handlers/restart_services.yml are smart enough to +# *conditionally* restart these services, only if they are installed on the +# system. +- name: setup basic /etc/fedmsg.d/ contents + template: > + src="{{ item }}.j2" + dest="/etc/fedmsg.d/{{ item }}" + owner=root + group=root + mode=644 + with_items: + - ssl.py + - endpoints.py + - relay.py + - logging.py + - base.py + tags: + - config + - fedmsgdconfig + notify: + - restart httpd + - restart fedmsg-relay + +- name: Remove unwanted files + file: dest=/etc/fedmsg.d/{{item}} state=absent + with_items: + - endpoints.py + tags: + - config + - fedmsgdconfig + notify: + - restart httpd + - restart fedmsg-relay + +- name: setup /etc/pki/fedmsg directory + file: path=/etc/pki/fedmsg owner=root group=root mode=0755 state=directory + tags: + - config + +- name: install fedmsg ca.cert + copy: > + src="{{ puppet_private }}/fedmsg-certs/keys/ca.crt" + dest=/etc/pki/fedmsg/ca.crt + owner=root + group=root + mode=0644 + tags: + - config + +- name: fedmsg certs + copy: > + src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(ansible_fqdn)}}.crt" + dest=/etc/pki/fedmsg/ + mode=644 + owner={{item['owner']}} + group={{item['group']}} + with_items: + - "{{ fedmsg_certs }}" + when: fedmsg_certs != [] + tags: + - config + +- name: fedmsg keys + copy: > + src="{{ private }}/files/fedmsg-certs/keys/{{item['service']}}-{{fedmsg_fqdn | default(ansible_fqdn)}}.key" + dest=/etc/pki/fedmsg/ + mode=0640 + owner={{item['owner']}} + group={{item['group']}} + with_items: + - "{{ fedmsg_certs }}" + when: fedmsg_certs != [] + tags: + - config + +# Three tasks for handling our custom selinux module +- name: ensure a directory exists for our custom selinux module + file: dest=/usr/local/share/fedmsg state=directory + +- name: copy over our custom selinux module + copy: src=selinux/fedmsg.pp dest=/usr/local/share/fedmsg/fedmsg.pp + register: selinux_module + +- name: install our custom selinux module + command: semodule -i /usr/local/share/fedmsg/fedmsg.pp + when: selinux_module|changed diff --git a/roles/anitya/fedmsg/templates/base.py.j2 b/roles/anitya/fedmsg/templates/base.py.j2 new file mode 100644 index 0000000000..b658188b19 --- /dev/null +++ b/roles/anitya/fedmsg/templates/base.py.j2 @@ -0,0 +1,83 @@ +config = dict( + # Set this to dev if you're hacking on fedmsg or an app locally. + # Set to stg or prod if running in the Fedora Infrastructure. + {% if env == 'staging' %} + environment="stg", + {% else %} + environment="prod", + {% endif %} + + {% if not ansible_hostname.startswith('busgateway') %} + # These options provide a place for hub processes to write out their last + # processed message. This let's them read it in at startup and figure out + # what kind of backlog they have to deal with. + {% if env == 'staging' %} + # But we have it turned off in staging. + #status_directory="/var/run/fedmsg/status", + {% else %} + status_directory="/var/run/fedmsg/status", + {% endif %} + + # This is the URL of a datagrepper instance that we can query for backlog. + {% if env == 'staging' %} + # But we have it turned off in staging. + #datagrepper_url="https://apps.stg.fedoraproject.org/datagrepper/raw", + {% else %} + datagrepper_url="https://apps.fedoraproject.org/datagrepper/raw", + {% endif %} + {% endif %} + + # This used to be set to 1 for safety, but it turns out it was + # excessive. It is the number of seconds that fedmsg should sleep + # after it has initialized, but before it begins to try and send any + # messages. If set to a non-zero value, this will slow down one-off + # fedmsg scripts like the git post-receive hook and pkgdb2branch. + # If we are experiencing message-loss problems, one of the first things + # to try should be to turn this number up to a non-zero value. '1' should + # be more than sufficient. + post_init_sleep=0.4, + + # This is the number of milliseconds to wait before timing out on + # connections.. notably to the fedmsg-relay in the event that it has + # crashed. + zmq_linger=2000, + + # Default is 0 + high_water_mark=0, + io_threads=1, + + # We almost always want the fedmsg-hub to be sending messages with zmq as + # opposed to amqp or stomp. The only exception will be the bugzilla + # amqp<->zmq bridge service. + zmq_enabled=True, + + # When subscribing to messages, we want to allow splats ('*') so we tell the + # hub to not be strict when comparing messages topics to subscription + # topics. + zmq_strict=False, + + # See the following + # - http://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html + # - http://api.zeromq.org/3-2:zmq-setsockopt + zmq_tcp_keepalive=1, + zmq_tcp_keepalive_cnt=3, + zmq_tcp_keepalive_idle=60, + zmq_tcp_keepalive_intvl=5, +) + +# This option adds an IPC socket by which we can monitor hub health. +try: + import os + import psutil + + pid = os.getpid() + proc = [p for p in psutil.process_iter() if p.pid == pid][0] + + config['moksha.monitoring.socket'] = \ + 'ipc:///var/run/fedmsg/monitoring-%s.socket' % proc.name + config['moksha.monitoring.socket.mode'] = '770' +except (OSError, ImportError): + # We run into issues when trying to import psutil from mod_wsgi on rhel7 + # but this feature is of no concern in that context, so just fail quietly. + # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837 + pass diff --git a/roles/anitya/fedmsg/templates/logging.py.j2 b/roles/anitya/fedmsg/templates/logging.py.j2 new file mode 100644 index 0000000000..b9f239cb8e --- /dev/null +++ b/roles/anitya/fedmsg/templates/logging.py.j2 @@ -0,0 +1,156 @@ +# Setup fedmsg logging. + +# All of these modules are just used by the ContextInjector below. +import inspect +import logging +import os +import socket +import traceback + +psutil = None +try: + import psutil +except (OSError, ImportError): + # We run into issues when trying to import psutil from inside mod_wsgi on + # rhel7. If we hit that here, then just fail quietly. + # https://github.com/jmflinuxtx/kerneltest-harness/pull/17#issuecomment-48007837 + pass + + +class ContextInjector(logging.Filter): + """ Logging filter that adds context to log records. + + Filters are typically used to "filter" log records. They declare a filter + method that can return True or False. Only records with 'True' will + actually be logged. + + Here, we somewhat abuse the concept of a filter. We always return true, + but we use the opportunity to hang important contextual information on the + log record to later be used by the logging Formatter. We don't normally + want to see all this stuff in normal log records, but we *do* want to see + it when we are emailed error messages. Seeing an error, but not knowing + which host it comes from, is not that useful. + + http://docs.python.org/2/howto/logging-cookbook.html#filters-contextual + """ + + def filter(self, record): + current_process = ContextInjector.get_current_process() + current_hostname = socket.gethostname() + + record.host = current_hostname + record.proc = current_process + record.pid = current_process.pid + record.proc_name = current_process.name + record.command_line = " ".join(current_process.cmdline) + record.callstack = self.format_callstack() + return True + + @staticmethod + def format_callstack(): + for i, frame in enumerate(f[0] for f in inspect.stack()): + if not '__name__' in frame.f_globals: + continue + modname = frame.f_globals['__name__'].split('.')[0] + if modname != "logging": + break + + def _format_frame(frame): + return ' File "%s", line %i in %s\n %s' % (frame) + + stack = traceback.extract_stack() + stack = stack[:-i] + return "\n".join([_format_frame(frame) for frame in stack]) + + @staticmethod + def get_current_process(): + mypid = os.getpid() + + if not psutil: + raise OSError("Could not import psutil for %r" % mypid) + + for proc in psutil.process_iter(): + if proc.pid == mypid: + return proc + + # This should be impossible. + raise ValueError("Could not find process %r" % mypid) + + @classmethod + def __json__(cls): + """ We need to be jsonifiable for "fedmsg-config" """ + return {'name': 'ContextInjector'} + + +hefty_format = """Message +------- +[%(asctime)s][%(name)10s %(levelname)7s] +%(message)s + +Process Details +--------------- +host: %(host)s +PID: %(pid)s +name: %(proc_name)s +command: %(command_line)s + +Callstack that lead to the logging statement +-------------------------------------------- +%(callstack)s +""" + + +# See the following for constraints on this format http://bit.ly/Xn1WDn +config = dict( + logging=dict( + version=1, + formatters=dict( + bare={ + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": "[%(asctime)s][%(name)10s %(levelname)7s] %(message)s" + }, + hefty={ + "datefmt": "%Y-%m-%d %H:%M:%S", + "format": hefty_format, + }, + ), + filters=dict( + context={ + # This "()" syntax in the stdlib doesn't seem to be documented + # anywhere. I had to read + # /usr/lib64/python2.7/logging/config.py to figure it out. + "()": ContextInjector, + }, + ), + handlers=dict( + console={ + "class": "logging.StreamHandler", + "formatter": "bare", + "level": "INFO", + "stream": "ext://sys.stdout", + }, + mailer={ + "class": "logging.handlers.SMTPHandler", + "formatter": "hefty", + "filters": ["context"], + "level": "ERROR", + "mailhost": "bastion.phx2.fedoraproject.org", + "fromaddr": "fedmsg@fedoraproject.org", + "toaddrs": ["sysadmin-datanommer-members@fedoraproject.org"], + "subject": "fedmsg error log", + }, + ), + loggers=dict( + fedmsg={ + "level": "INFO", + "propagate": False, + "handlers": ["console", "mailer"], + }, + moksha={ + "level": "INFO", + "propagate": False, + "handlers": ["console", "mailer"], + }, + ), + ), +) diff --git a/roles/anitya/fedmsg/templates/relay.py.j2 b/roles/anitya/fedmsg/templates/relay.py.j2 new file mode 100644 index 0000000000..2f4210568e --- /dev/null +++ b/roles/anitya/fedmsg/templates/relay.py.j2 @@ -0,0 +1,16 @@ +config = dict( + endpoints={ + # This is the output side of the relay to which all other + # services can listen. + "relay_outbound": [ + # Messages emerge here + "tcp://release-monitoring.org:9940', + ], + }, + + # wsgi scripts on the frontend talk back here + # so do cronjobs on anitya-backend01. they get a firewall rule. + relay_inbound=[ + "tcp://anitya-frontend01.fedoraproject.org:9941", + ], +) diff --git a/roles/anitya/fedmsg/templates/ssl.py.j2 b/roles/anitya/fedmsg/templates/ssl.py.j2 new file mode 100644 index 0000000000..2a55f4e1a8 --- /dev/null +++ b/roles/anitya/fedmsg/templates/ssl.py.j2 @@ -0,0 +1,192 @@ + +{% if env == 'staging' %} +suffix = "stg.phx2.fedoraproject.org" +topic_prefix = "org.fedoraproject.stg." +{% else %} +suffix = "phx2.fedoraproject.org" +topic_prefix = "org.fedoraproject.prod." +{% endif %} + +vpn_suffix = "vpn.fedoraproject.org" + +config = dict( + sign_messages=True, + validate_signatures=True, + ssldir="/etc/pki/fedmsg", + + {% if env == 'staging' %} + # In staging, we use the internal IP of proxy01 without ssl. + crl_location="http://10.5.126.88/fedmsg/crl.pem", + {% else %} + crl_location="https://fedoraproject.org/fedmsg/crl.pem", + {% endif %} + crl_cache="/var/run/fedmsg/crl.pem", + crl_cache_expiry=86400, # Daily + + certnames=dict( + [ + ("bugzilla.bugzilla2fedmsg01", "bugzilla2fedmsg-bugzilla2fedmsg01.%s" % suffix) + ] + [ + ("shell.bugzilla2fedmsg01", "shell-bugzilla2fedmsg01.%s" % suffix) + ] + [ + ("github2fedmsg.github2fedmsg0%i" % i, "github2fedmsg-github2fedmsg0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.github2fedmsg0%i" % i, "shell-github2fedmsg0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("fedimg.fedimg01", "fedimg-fedimg01.%s" % suffix), + ("shell.fedimg01", "shell-fedimg01.%s" % suffix), + ] + [ + ("kerneltest.kerneltest0%i" % i, "kerneltest-kerneltest0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.kerneltest0%i" % i, "shell-kerneltest0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.notifs-web0%i" % i, "shell-notifs-web0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("fmn.notifs-web0%i" % i, "fmn-notifs-web0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("fmn.notifs-backend01", "fmn-notifs-backend01.%s" % suffix), + ] + [ + ("shell.pkgdb0%i" % i, "shell-pkgdb0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("pkgdb2.pkgdb0%i" % i, "pkgdb-pkgdb0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("mediawiki.wiki0%i" % i, "mediawiki-wiki0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.fas0%i" % i, "shell-fas0%i.%s" % (i, suffix)) + for i in range(1, 4) + ] + [ + ("fas.fas0%i" % i, "fas-fas0%i.%s" % (i, suffix)) + for i in range(1, 4) + ] + [ + ("shell.packages0%i" % i, "shell-packages0%i.%s" % (i, suffix)) + for i in range(3, 5) + ] + [ + ("fedoratagger.tagger0%i" % i, "fedoratagger-tagger0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.tagger0%i" % i, "shell-tagger0%i.%s" % (i, suffix)) + for i in range(1, 3) + ] + [ + ("shell.pkgs0%i" % i, "shell-pkgs0%i.%s" % (i, suffix)) + for i in range(1, 2) + ] + [ + ("scm.pkgs0%i" % i, "scm-pkgs0%i.%s" % (i, suffix)) + for i in range(1, 2) + ] + [ + ("lookaside.pkgs0%i" % i, "lookaside-pkgs0%i.%s" % (i, suffix)) + for i in range(1, 2) + ] + [ + ("shell.relepel01", "shell-relepel01.%s" % suffix), + ("shell.branched-composer", "shell-branched-composer.%s" % suffix), + ("shell.rawhide-composer", "shell-rawhide-composer.%s" % suffix), + ("shell.releng03", "shell-releng03.%s" % suffix), + ("shell.releng04", "shell-releng04.%s" % suffix), + ("bodhi.relepel01", "bodhi-relepel01.%s" % suffix), + ("bodhi.branched-composer", "bodhi-branched-composer.%s" % suffix), + ("bodhi.rawhide-composer", "bodhi-rawhide-composer.%s" % suffix), + ("bodhi.releng03", "bodhi-releng03.%s" % suffix), + ("bodhi.releng04", "bodhi-releng04.%s" % suffix), + ] + [ + ("ftpsync.relepel01", "ftpsync-relepel01.%s" % suffix), + ("ftpsync.releng04", "ftpsync-releng04.%s" % suffix), + ] + [ + ("shell.busgateway01", "shell-busgateway01.%s" % suffix), + ] + [ + ("shell.value01", "shell-value01.%s" % suffix), + ("supybot.value01", "supybot-value01.%s" % suffix), + ] + [ + ("koji.koji04", "koji-koji04.%s" % suffix), + ("koji.koji01", "koji-koji01.%s" % suffix), + ("koji.koji03", "koji-koji03.%s" % suffix), + ("shell.koji04", "shell-koji04.%s" % suffix), + ("shell.koji01", "shell-koji01.%s" % suffix), + ("shell.koji03", "shell-koji03.%s" % suffix), + ] + [ + ("nagios.noc01", "nagios-noc01.%s" % suffix), + ("shell.noc01", "shell-noc01.%s" % suffix), + ] + [ + ("git.hosted03", "git-hosted03.%s" % vpn_suffix), + ("git.hosted04", "git-hosted04.%s" % vpn_suffix), + ("trac.hosted03", "trac-hosted03.%s" % vpn_suffix), + ("trac.hosted04", "trac-hosted04.%s" % vpn_suffix), + ("shell.hosted03", "shell-hosted03.%s" % vpn_suffix), + ("shell.hosted04", "shell-hosted04.%s" % vpn_suffix), + ] + [ + ("shell.lockbox01", "shell-lockbox01.%s" % suffix), + ("announce.lockbox01", "announce-lockbox01.%s" % suffix), + ] + [ + # These first two entries are here to placate a bug in + # python-askbot-fedmsg-0.0.4. They can be removed once + # python-askbot-fedmsg-0.0.5 hits town. + ("askbot.ask01.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix), + ("askbot.ask01.stg.phx2.fedoraproject.org", "askbot-ask01.%s" % suffix), + + ("askbot.ask01", "askbot-ask01.%s" % suffix), + ("shell.ask01", "shell-ask01.%s" % suffix), + + ("askbot.ask02", "askbot-ask02.%s" % suffix), + ("shell.ask02", "shell-ask02.%s" % suffix), + + ("fedbadges.badges-backend01", "fedbadges-badges-backend01.%s" % suffix), + ("shell.badges-backend01", "shell-badges-backend01.%s" % suffix), + + ("summershum.summershum01", "summershum-summershum01.%s" % suffix), + ("shell.summershum01", "shell-summershum01.%s" % suffix), + + ("tahrir.badges-web01", "tahrir-badges-web01.%s" % suffix), + ("shell.badges-web01", "shell-badges-web01.%s" % suffix), + ("tahrir.badges-web02", "tahrir-badges-web02.%s" % suffix), + ("shell.badges-web02", "shell-badges-web02.%s" % suffix), + + ("shell.nuancier01", "shell-nuancier01.%s" % suffix), + ("shell.nuancier02", "shell-nuancier02.%s" % suffix), + ("nuancier.nuancier01", "nuancier-nuancier01.%s" % suffix), + ("nuancier.nuancier02", "nuancier-nuancier02.%s" % suffix), + + ("shell.fedocal01", "shell-fedocal01.%s" % suffix), + ("shell.fedocal02", "shell-fedocal02.%s" % suffix), + ("fedocal.fedocal01", "fedocal-fedocal01.%s" % suffix), + ("fedocal.fedocal02", "fedocal-fedocal02.%s" % suffix), + + ("shell.mailman01", "shell-mailman01.%s" % suffix), + ("mailman.mailman01", "mailman-mailman01.%s" % suffix), + + ("shell.bodhi01", "shell-bodhi01.%s" % suffix), + ("shell.bodhi02", "shell-bodhi02.%s" % suffix), + ("bodhi.bodhi01", "bodhi-bodhi01.%s" % suffix), + ("bodhi.bodhi02", "bodhi-bodhi02.%s" % suffix), + + ("shell.elections01", "shell-elections01.%s" % suffix), + ("shell.elections02", "shell-elections02.%s" % suffix), + ("fedora_elections.elections01", "fedora_elections-elections01.%s" % suffix), + ("fedora_elections.elections02", "fedora_elections-elections02.%s" % suffix), + + ("shell.anitya-frontend01", "shell-anitya-frontend01.fedoraproject.org"), + ("anitya.anitya-frontend01", "anitya-anitya-frontend01.fedoraproject.org"), + ("shell.anitya-backend01", "shell-anitya-backend01.fedoraproject.org"), + ("anitya.anitya-backend01", "anitya-anitya-backend01.fedoraproject.org"), + + # This is for the copr backend, which is a little different. The + # "cert-prefix" is just "copr", and is hardcoded in + # backend/dispatcher.py. The hostname is also a little different, + # since it is an openstack node. This might be a little fragile. :/ + # See https://github.com/fedora-infra/fedmsg/issues/199 for the plan. + ("copr.dhcp-client03", "copr-copr-be.cloud.fedoraproject.org"), + ("copr.copr-be-i-00000407", "copr-copr-be.cloud.fedoraproject.org"), + ("copr.copr-be", "copr-copr-be.cloud.fedoraproject.org"), + + # Jenkins, also being a cloud node, is weird. Like copr. + ("shell.jenkins-master-unknown", "shell-jenkins.cloud.fedoraproject.org"), + ("jenkins.jenkins-master-unknown", "jenkins-jenkins.cloud.fedoraproject.org"), + ]), +) +