Hubs: update the playbook

playbooks/hosts/hubs-dev.fedorainfracloud.org.yml

@@ -55,10 +55,12 @@
     main_user: fedora
     hubs_url_hostname: "{{ ansible_fqdn }}"
     hubs_secret_key: demotestinghubsmachine
-    hubs_db_type: sqlite
+    hubs_db_type: postgresql
     hubs_dev_mode: false
     hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem
     hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
+    hubs_fas_username: "{{ fedoraDummyUser }}",
+    hubs_fas_password: "{{ fedoraDummyUserPassword }}",
 
 
   tasks:

roles/hubs/defaults/main.yml

@@ -8,7 +8,9 @@ hubs_venv_dir: "{{ hubs_base_dir }}/venv"
 hubs_var_dir: "{{ hubs_base_dir }}/var"
 hubs_db_type: sqlite
 hubs_db_password: changeme
-hubs_url_hostname: localhost
+hubs_url_hostname: "{{ ansible_fqdn }}"
 hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %}
-hubs_ssl_cert: /etc/pki/tls/certs/localhost.crt
+hubs_ssl_cert: /etc/pki/tls/certs/{{ hubs_url_hostname }}.crt
-hubs_ssl_key: /etc/pki/tls/private/localhost.key
+hubs_ssl_key: /etc/pki/tls/private/{{ hubs_url_hostname }}.key
+hubs_fas_username: changeme
+hubs_fas_password: changeme

roles/hubs/files/logging.ini

@@ -1,4 +1,3 @@
-# From https://docs.python.org/2/howto/logging.html
 [loggers]
 keys=root
 
@@ -6,7 +5,7 @@ keys=root
 keys=console
 
 [formatters]
-keys=simple
+keys=simple,minimal
 
 [logger_root]
 level=DEBUG
@@ -15,9 +14,13 @@ handlers=console
 [handler_console]
 class=StreamHandler
 level=DEBUG
-formatter=simple
+formatter=minimal
 args=(sys.stdout,)
 
 [formatter_simple]
 format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s
 datefmt=%H:%M:%S
+
+[formatter_minimal]
+format=[%(levelname)s] (%(name)s) %(message)s
+datefmt=%H:%M:%S

roles/hubs/handlers/main.yml

@@ -4,18 +4,33 @@
 - name: restart the hubs-specific fedmsg-hub
   service: name=hubs-fedmsg-hub state=restarted
   listen: "hubs configuration change"
+  when: not hubs_dev_mode
+
+- name: restart the hubs-specific fedmsg-relay
+  service: name=hubs-fedmsg-relay state=restarted
+  listen: "hubs configuration change"
+  when: not hubs_dev_mode
 
 - name: restart hubs triage
   service: name=hubs-triage@* state=restarted
   listen: "hubs configuration change"
+  when: not hubs_dev_mode
 
 - name: restart hubs workers
   service: name=hubs-worker@* state=restarted
   listen: "hubs configuration change"
+  when: not hubs_dev_mode
 
 - name: restart hubs SSE server
   service: name=hubs-sse state=restarted
   listen: "hubs configuration change"
+  when: not hubs_dev_mode
 
 # Webserver
-- import_tasks: webserver.yml
+- name: restart hubs webapp
+  service: name=hubs-webapp state=restarted
+  listen: "hubs configuration change"
+  when: not hubs_dev_mode
+
+- name: restart nginx
+  service: name=nginx state=restarted

roles/hubs/handlers/webserver.yml

@@ -1,7 +0,0 @@
-- name: restart hubs webapp
-  service: name=hubs-webapp state=restarted
-  listen: "hubs configuration change"
-  when: not hubs_dev_mode
-
-- name: restart nginx
-  service: name=nginx state=restarted

roles/hubs/tasks/db-postgresql.yml

@@ -52,5 +52,5 @@
   environment:
     HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py"
   become_user: "{{ main_user }}"
-  when: db_creation|succeeded
+  when: db_creation|succeeded and db_creation|changed
 

roles/hubs/tasks/dev.yml

@@ -0,0 +1,30 @@
+- name: Install Fedora Hubs development packages
+  dnf: name={{ item }} state=present
+  with_items:
+    - gcc
+    - gcc-c++
+    - libffi-devel
+    - openssl-devel
+    - python-sphinx
+    - python2-devel
+    - python3-devel
+    - python3-honcho
+    - python3-tox
+    - redhat-rpm-config
+    - sqlite-devel
+
+- name: Install a custom bashrc
+  template: src=bashrc dest=/home/{{ main_user }}/.bashrc
+
+- name: Install Honcho's env file
+  template: src=honcho-env dest={{ hubs_base_dir }}/.env
+
+- name: Install Honcho's procfile
+  template: src=honcho-procfile dest={{ hubs_base_dir }}/Procfile
+
+- name: Link to the FAS credentials file if any
+  file:
+    state: link
+    path: "/etc/fedmsg.d/fas_credentials.py"
+    src: "{{ hubs_code_dir }}/fedmsg.d/fas_credentials.py"
+  notify: "hubs configuration change"

roles/hubs/tasks/main.yml

@@ -11,54 +11,33 @@
     - npm
     - redis
     - fedmsg-hub
-    - python-virtualenv
+    - fedmsg-relay
+    - python3-virtualenv
     - python3-flask-oidc
-
+    - postfix
-- name: Install Fedora Hubs development packages
-  dnf: name={{ item }} state=present
-  with_items:
-    - gcc
-    - gcc-c++
-    - libffi-devel
-    - openssl-devel
-    - python-sphinx
-    - python2-devel
-    - python3-devel
-    - redhat-rpm-config
-    - sqlite-devel
-  when: hubs_dev_mode
 
 - name: Install the distribution versions of requirements.txt
   dnf: name={{ item }} state=present
   with_items:
-    - python-alembic
+    - python3-alembic
-    - python-arrow
+    - python3-arrow
-    - python-bleach
+    - python3-bleach
-    - python-decorator
+    - python3-decorator
-    - python-dogpile-cache
+    - python3-dogpile-cache
-    - python-fedmsg-core
+    - python3-fedmsg-core
-    - python-fedmsg-meta-fedora-infrastructure
+    - python3-fedmsg-meta-fedora-infrastructure
-    - python-flask
+    - python3-flask
-    - python-flask-oidc
+    - python3-flask-oidc
-    - python-fmn-lib
+    - python3-html5lib
-    - python-fmn-rules
+    - python3-munch
-    - python-futures
+    - python3-pytz
-    - python-html5lib
+    - python3-sqlalchemy
-    - python-munch
+    - python3-markdown
-    - pytz
+    - python3-pkgwat-api
-    - python-sqlalchemy
+    - python3-six
-    - python-markdown
+    - python3-pygments
-    - python2-pkgwat-api
+    - python3-pygments-markdown-lexer
-    - python-six
+    - python3-retask
-    - python-pygments
-    - python-pygments-markdown-lexer
-    - python-retask
-
-
-# Add various helpful configuration files
-- name: Install a custom bashrc
-  template: src=bashrc dest=/home/{{ main_user }}/.bashrc
-  when: hubs_dev_mode
 
 
 # Create directory structure
@@ -84,6 +63,7 @@
     requirements: "{{ hubs_code_dir }}/requirements.txt"
     virtualenv: "{{ hubs_venv_dir}}"
     virtualenv_site_packages: yes
+    virtualenv_command: virtualenv-3
 
 - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv
   become_user: "{{ main_user }}"
@@ -91,6 +71,7 @@
     requirements: "{{ hubs_code_dir }}/test-requirements.txt"
     virtualenv: "{{ hubs_venv_dir}}"
     virtualenv_site_packages: yes
+    virtualenv_command: virtualenv-3
 
 - name: Install other packages into hubs virtualenv
   become_user: "{{ main_user }}"
@@ -98,6 +79,7 @@
     name: "{{ item }}"
     virtualenv: "{{ hubs_venv_dir }}"
     virtualenv_site_packages: yes
+    virtualenv_command: virtualenv-3
   with_items:
     - bleach
 
@@ -105,7 +87,7 @@
   become_user: "{{ main_user }}"
   command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}"
   args:
-    creates: "{{ hubs_venv_dir }}/lib/python2.7/site-packages/fedora-hubs.egg-link"
+    creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link"
 
 - name: Set bin file context in the virtualenv
   become_user: "{{ main_user }}"
@@ -144,8 +126,14 @@
     creates: "{{ hubs_conf_dir }}/client_secrets.json"
 
 
+- name: Start and enable the common services
+  service: name={{ item }} state=started enabled=yes
+  with_items:
+    - redis
+    - postfix
+
 # Set up, create, and populate the database.
-- import_tasks: db-{{ hubs_db_type }}.yml
+- include_tasks: db-{{ hubs_db_type }}.yml
 
 
 # Set up JavaScript requirements
@@ -156,7 +144,7 @@
     creates: node_modules
     chdir: "{{ hubs_code_dir }}/hubs/static/client"
 
-- name: Build JavaScript assests
+- name: Build JavaScript assets
   command: npm run build
   become_user: "{{ main_user }}"
   args:
@@ -164,51 +152,18 @@
     creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js"
 
 
-- name: Fix permissions if necessary
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ main_user }}"
-    group: "{{ main_user }}"
-    recurse: yes
-    #setype: httpd_sys_content_rw_t
-  with_items:
-    - "{{ hubs_base_dir }}"
-    - "{{ hubs_conf_dir }}"
-    - "{{ hubs_var_dir }}"
-
-
 # Services
-- name: Disable the system-wide fedmsg-hub
+- name: Disable the system-wide fedmsg daemons
-  service: name=fedmsg-hub state=stopped enabled=no
+  service: name={{ item }} state=stopped enabled=no
-
-- name: Install the service files
-  template:
-    src: "{{ item }}.service"
-    dest: /etc/systemd/system/{{ item }}.service
   with_items:
-    - hubs-triage@
+    - fedmsg-hub
-    - hubs-worker@
+    - fedmsg-relay
-    - hubs-sse
-    - hubs-fedmsg-hub
-  register: service_installed
-
-- name: reload systemd
-  command: systemctl daemon-reload
-  when: service_installed|changed
-
-- name: Start and enable the services
-  service: name={{ item }} state=started enabled=yes
-  with_items:
-    - redis
-    - hubs-triage@1
-    - hubs-triage@2
-    - hubs-worker@1
-    - hubs-worker@2
-    - hubs-sse
-    - hubs-fedmsg-hub
 
 
-# Webserver
+# Include mode-specific tasks
-- import_tasks: webserver.yml
+
+- include_tasks: dev.yml
+  when: hubs_dev_mode
+
+- include_tasks: prod.yml
   when: not hubs_dev_mode

roles/hubs/tasks/prod.yml

@@ -0,0 +1,28 @@
+- name: Install the service files
+  template:
+    src: "{{ item }}.service"
+    dest: /etc/systemd/system/{{ item }}.service
+  with_items:
+    - hubs-triage@
+    - hubs-worker@
+    - hubs-sse
+    - hubs-fedmsg-hub
+    - hubs-fedmsg-relay
+  register: service_installed
+
+- name: reload systemd
+  command: systemctl daemon-reload
+  when: service_installed|changed
+
+- name: Start and enable the services in prod mode
+  service: name={{ item }} state=started enabled=yes
+  with_items:
+    - hubs-triage@1
+    - hubs-triage@2
+    - hubs-worker@1
+    - hubs-worker@2
+    - hubs-sse
+    - hubs-fedmsg-hub
+    - hubs-fedmsg-relay
+
+- include_tasks: webserver.yml

roles/hubs/tasks/webserver.yml

@@ -8,6 +8,17 @@
     - libsemanage-python
 
 
+- name: install python2-certbot-nginx
+  dnf: name=python2-certbot-nginx state=present
+
+- name: get the letencrypt cert
+  command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org
+  args:
+    creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
+  notify:
+    - restart nginx
+
+
 - name: Gunicorn logging configuration
   copy:
     src: logging.ini
@@ -37,17 +48,6 @@
     - restart nginx
 
 
-- name: install python2-certbot-nginx
-  dnf: name=python2-certbot-nginx state=present
-
-- name: get the letencrypt cert
-  command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org
-  args:
-    creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
-  notify:
-    - restart nginx
-
-
 - name: Nginx proxy configuration
   copy:
     src: "{{ item }}"

roles/hubs/templates/bashrc

@@ -13,6 +13,10 @@ fi
 # by defining a variable with name _<function>_help containing the help text
 
 
+# Honcho has issues outputing UTF-8 in Vagrant SSH
+# https://github.com/nickstenning/honcho/issues/51
+export PYTHONIOENCODING=utf-8
+
 export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
 export FLASK_APP={{ hubs_code_dir }}/hubs/app.py
 
@@ -23,14 +27,9 @@ workon() {
     cd {{ hubs_code_dir }}
 }
 
-hup() {
+alias hup="pushd ~ ; honcho start ; popd"
-    source {{ hubs_venv_dir }}/bin/activate
-    pushd {{ hubs_code_dir }}
-    FLASK_DEBUG=1 flask run --host 0.0.0.0 --port 5000
-}
 
 hreset() {
-    source {{ hubs_venv_dir }}/bin/activate
     {% if hubs_db_type == "postgresql" %}
     sudo -u postgres dropdb hubs
     sudo -u postgres createdb -O hubs hubs
@@ -39,7 +38,6 @@ hreset() {
     {% endif %}
     rm {{ hubs_var_dir }}/cache.db
     pushd {{ hubs_code_dir }}
-    python populate.py
+    {{ hubs_venv_dir }}/bin/python populate.py
     popd
-    deactivate
 }

roles/hubs/templates/fedmsg_config

@@ -20,10 +20,13 @@ config = {
     'hubs.consumer.enabled': True,
     'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue',
 
-    # FAS
+    # Use fedmsg-relay to publish messages
+    'active': True,
+
+    # FAS credentials
     'fas_credentials': {
-        'username': '{{ fedoraDummyUser }}',
+        'username': '{{ hubs_fas_username }}',
-        'password': '{{ fedoraDummyUserPassword }}',
+        'password': '{{ hubs_fas_password }}',
     },
 }
 

roles/hubs/templates/honcho-env

@@ -0,0 +1,3 @@
+FLASK_DEBUG=1
+FLASK_APP={{ hubs_code_dir }}/hubs/app.py
+HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py

roles/hubs/templates/honcho-procfile

@@ -0,0 +1,7 @@
+web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask-3 run --host 0.0.0.0 --port 5000
+triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage
+worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker
+sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac
+fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub
+fedmsg_relay: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay
+js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev

roles/hubs/templates/hubs-fedmsg-relay.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Hubs-specific fedmsg processing relay
+After=network.target
+Documentation=https://fedmsg.readthedocs.org/
+
+[Service]
+ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay
+Type=simple
+User=fedmsg
+Group=fedmsg
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target