Toy with dynamically generating fedmsg endpoint config.

inventory/group_vars/all

@@ -63,6 +63,14 @@ fedmsg_active: False
 fedmsg_prefix: org.fedoraproject
 fedmsg_env: prod
 
+# These are used to:
+#  1) configure mod_wsgi
+#  2) open iptables rules for fedmsg (per wsgi thread)
+#  3) declare enough fedmsg endpoints for the service
+#wsgi_fedmsg_service:  bodhi
+#wsgi_procs: 4
+#wsgi_threads: 4
+
 # By default, nodes don't backup any dbs on them unless they declare it.
 dbs_to_backup: []
 

inventory/group_vars/badges-web

@@ -4,13 +4,15 @@ mem_size: 4096
 num_cpus: 2
 freezes: false
 
-# for systems that do not match the above - specify the same parameter in
-# the host_vars/$hostname file
+# Definining these vars has a number of effects
+# 1) mod_wsgi is configured to use the vars for its own setup
+# 2) iptables opens enough ports for all threads for fedmsg
+# 3) roles/fedmsg/base/ declares enough fedmsg endpoints for all threads
+wsgi_fedmsg_service: tahrir
+wsgi_procs: 4
+wsgi_threads: 4
 
-tcp_ports: [ 80, 443,
-    # These 16 ports are used by fedmsg.  One for each wsgi thread.
-    3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007,
-    3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015]
+tcp_ports: [ 80 ]
 
 # Neeed for rsync from log01 for logs.
 custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT' ]

inventory/group_vars/badges-web-stg

@@ -4,13 +4,15 @@ lvm_size: 20000
 mem_size: 1024
 num_cpus: 2
 
-# for systems that do not match the above - specify the same parameter in
-# the host_vars/$hostname file
+# Definining these vars has a number of effects
+# 1) mod_wsgi is configured to use the vars for its own setup
+# 2) iptables opens enough ports for all threads for fedmsg
+# 3) roles/fedmsg/base/ declares enough fedmsg endpoints for all threads
+wsgi_fedmsg_service: tahrir
+wsgi_procs: 4
+wsgi_threads: 4
 
-tcp_ports: [ 80, 443,
-    # These 16 ports are used by fedmsg.  One for each wsgi thread.
-    3000, 3001, 3002, 3003, 3004, 3005, 3006, 3007,
-    3008, 3009, 3010, 3011, 3012, 3013, 3014, 3015]
+tcp_ports: [ 80 ]
 
 # Neeed for rsync from log01 for logs.
 custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.13 --dport 873 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 192.168.1.59 --dport 873 -j ACCEPT' ]

roles/badges/frontend/tasks/main.yml

@@ -41,7 +41,7 @@
   - restart apache
 
 - name: copy tahrir httpd config
-  copy: >
+  template: >
     src={{ item }} dest="/etc/httpd/conf.d/{{ item }}"
     owner=apache group=apache mode=0644
   with_items:

roles/badges/frontend/templates/tahrir.conf

@@ -2,7 +2,7 @@ Alias /static /usr/lib/python2.7/site-packages/tahrir/static
 Alias /pngs /usr/share/badges/pngs
 Alias /stls /usr/share/badges/stls
 
-WSGIDaemonProcess tahrir user=tahrir group=tahrir maximum-requests=1000 display-name=tahrir processes=4 threads=4
+WSGIDaemonProcess tahrir user=tahrir group=tahrir maximum-requests=1000 display-name=tahrir processes={{ wsgi_procs }} threads={{ wsgi_threads }}
 WSGISocketPrefix run/wsgi
 WSGIRestrictStdout On
 WSGIRestrictSignal Off

roles/base/templates/iptables/iptables

@@ -40,6 +40,13 @@
 {% endfor %}
 {% endif %}
 
+# if the host declares a fedmsg-enabled wsgi app, open ports for it
+{% if fedmsg_wsgi_service is defined %}
+{% for i in range(wsgi_procs * wsgi_threads) %}
+-A INPUT -p tcp -m tcp --dport 30{{ '%02d' % i }} -j ACCEPT
+{% endfor %}
+{% endif %}
+
 # if the host/group defines incoming tcp_ports - allow them
 {% if tcp_ports is defined %}
 {% for port in tcp_ports %}

roles/fedmsg/base/templates/endpoints-fedbadges.py.j2

@@ -12,17 +12,5 @@ config = dict(
             "tcp://badges-backend01.%s:3002" % suffix,
             "tcp://badges-backend01.%s:3003" % suffix,
         ],
-
-        "tahrir.badges-web01": [
-            "tcp://badges-web01.%s:30%02i" % (suffix, i)
-            for i in range(16)
-        ],
-{% if env != 'staging' %}
-        "tahrir.badges-web02": [
-            "tcp://badges-web02.%s:30%02i" % (suffix, i)
-            for i in range(16)
-        ],
-{% endif %}
-
     },
 )

roles/fedmsg/base/templates/endpoints.py.j2

@@ -99,5 +99,18 @@ config = dict(
         ],
 
         # koji is not listed here since it publishes to the fedmsg-relay
+
+
+# Dynamically generate endpoint declarations from our wsgi app vars.
+# Eventually, replace *all* fedmsg endpoint definitions with this one loop
+{% for host in groups['all'] %}
+{% if 'fedmsg_wsgi_service' in hostvars[host] %}
+        "{{hostvars[host]['fedmsg_wsgi_service']}}.{{hostvars[host].split('.')|first}}": [
+{% for i in range(wsgi_procs * wsgi_threads) %}
+            "tcp://{{host}}:30{{%02d % i}}",
+{% endfor %}
+        ],
+{% endif %}
+{% endfor %}
     },
 )