diff --git a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml index 6d1b6265ed..36386dd71a 100644 --- a/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml +++ b/playbooks/hosts/hubs-dev.fedorainfracloud.org.yml @@ -55,10 +55,12 @@ main_user: fedora hubs_url_hostname: "{{ ansible_fqdn }}" hubs_secret_key: demotestinghubsmachine - hubs_db_type: sqlite + hubs_db_type: postgresql hubs_dev_mode: false hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem + hubs_fas_username: "{{ fedoraDummyUser }}", + hubs_fas_password: "{{ fedoraDummyUserPassword }}", tasks: diff --git a/roles/hubs/defaults/main.yml b/roles/hubs/defaults/main.yml index 2ee32214e4..f3d14cb644 100644 --- a/roles/hubs/defaults/main.yml +++ b/roles/hubs/defaults/main.yml @@ -8,7 +8,9 @@ hubs_venv_dir: "{{ hubs_base_dir }}/venv" hubs_var_dir: "{{ hubs_base_dir }}/var" hubs_db_type: sqlite hubs_db_password: changeme -hubs_url_hostname: localhost +hubs_url_hostname: "{{ ansible_fqdn }}" hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %} -hubs_ssl_cert: /etc/pki/tls/certs/localhost.crt -hubs_ssl_key: /etc/pki/tls/private/localhost.key +hubs_ssl_cert: /etc/pki/tls/certs/{{ hubs_url_hostname }}.crt +hubs_ssl_key: /etc/pki/tls/private/{{ hubs_url_hostname }}.key +hubs_fas_username: changeme +hubs_fas_password: changeme diff --git a/roles/hubs/files/logging.ini b/roles/hubs/files/logging.ini index 3512fa6ee9..f1a3bc0ddd 100644 --- a/roles/hubs/files/logging.ini +++ b/roles/hubs/files/logging.ini @@ -1,4 +1,3 @@ -# From https://docs.python.org/2/howto/logging.html [loggers] keys=root @@ -6,7 +5,7 @@ keys=root keys=console [formatters] -keys=simple +keys=simple,minimal [logger_root] level=DEBUG @@ -15,9 +14,13 @@ handlers=console [handler_console] class=StreamHandler level=DEBUG -formatter=simple +formatter=minimal args=(sys.stdout,) [formatter_simple] format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s datefmt=%H:%M:%S + +[formatter_minimal] +format=[%(levelname)s] (%(name)s) %(message)s +datefmt=%H:%M:%S diff --git a/roles/hubs/handlers/main.yml b/roles/hubs/handlers/main.yml index c6935af51c..f71ee8fd8c 100644 --- a/roles/hubs/handlers/main.yml +++ b/roles/hubs/handlers/main.yml @@ -4,18 +4,33 @@ - name: restart the hubs-specific fedmsg-hub service: name=hubs-fedmsg-hub state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode + +- name: restart the hubs-specific fedmsg-relay + service: name=hubs-fedmsg-relay state=restarted + listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs triage service: name=hubs-triage@* state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs workers service: name=hubs-worker@* state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode - name: restart hubs SSE server service: name=hubs-sse state=restarted listen: "hubs configuration change" + when: not hubs_dev_mode # Webserver -- import_tasks: webserver.yml +- name: restart hubs webapp + service: name=hubs-webapp state=restarted + listen: "hubs configuration change" + when: not hubs_dev_mode + +- name: restart nginx + service: name=nginx state=restarted diff --git a/roles/hubs/handlers/webserver.yml b/roles/hubs/handlers/webserver.yml deleted file mode 100644 index e6cb871791..0000000000 --- a/roles/hubs/handlers/webserver.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: restart hubs webapp - service: name=hubs-webapp state=restarted - listen: "hubs configuration change" - when: not hubs_dev_mode - -- name: restart nginx - service: name=nginx state=restarted diff --git a/roles/hubs/tasks/db-postgresql.yml b/roles/hubs/tasks/db-postgresql.yml index e8a560105c..2a9eec915c 100644 --- a/roles/hubs/tasks/db-postgresql.yml +++ b/roles/hubs/tasks/db-postgresql.yml @@ -52,5 +52,5 @@ environment: HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py" become_user: "{{ main_user }}" - when: db_creation|succeeded + when: db_creation|succeeded and db_creation|changed diff --git a/roles/hubs/tasks/dev.yml b/roles/hubs/tasks/dev.yml new file mode 100644 index 0000000000..6b01c9b8fc --- /dev/null +++ b/roles/hubs/tasks/dev.yml @@ -0,0 +1,30 @@ +- name: Install Fedora Hubs development packages + dnf: name={{ item }} state=present + with_items: + - gcc + - gcc-c++ + - libffi-devel + - openssl-devel + - python-sphinx + - python2-devel + - python3-devel + - python3-honcho + - python3-tox + - redhat-rpm-config + - sqlite-devel + +- name: Install a custom bashrc + template: src=bashrc dest=/home/{{ main_user }}/.bashrc + +- name: Install Honcho's env file + template: src=honcho-env dest={{ hubs_base_dir }}/.env + +- name: Install Honcho's procfile + template: src=honcho-procfile dest={{ hubs_base_dir }}/Procfile + +- name: Link to the FAS credentials file if any + file: + state: link + path: "/etc/fedmsg.d/fas_credentials.py" + src: "{{ hubs_code_dir }}/fedmsg.d/fas_credentials.py" + notify: "hubs configuration change" diff --git a/roles/hubs/tasks/main.yml b/roles/hubs/tasks/main.yml index a06608abee..db877d2c8d 100644 --- a/roles/hubs/tasks/main.yml +++ b/roles/hubs/tasks/main.yml @@ -11,54 +11,33 @@ - npm - redis - fedmsg-hub - - python-virtualenv + - fedmsg-relay + - python3-virtualenv - python3-flask-oidc - -- name: Install Fedora Hubs development packages - dnf: name={{ item }} state=present - with_items: - - gcc - - gcc-c++ - - libffi-devel - - openssl-devel - - python-sphinx - - python2-devel - - python3-devel - - redhat-rpm-config - - sqlite-devel - when: hubs_dev_mode + - postfix - name: Install the distribution versions of requirements.txt dnf: name={{ item }} state=present with_items: - - python-alembic - - python-arrow - - python-bleach - - python-decorator - - python-dogpile-cache - - python-fedmsg-core - - python-fedmsg-meta-fedora-infrastructure - - python-flask - - python-flask-oidc - - python-fmn-lib - - python-fmn-rules - - python-futures - - python-html5lib - - python-munch - - pytz - - python-sqlalchemy - - python-markdown - - python2-pkgwat-api - - python-six - - python-pygments - - python-pygments-markdown-lexer - - python-retask - - -# Add various helpful configuration files -- name: Install a custom bashrc - template: src=bashrc dest=/home/{{ main_user }}/.bashrc - when: hubs_dev_mode + - python3-alembic + - python3-arrow + - python3-bleach + - python3-decorator + - python3-dogpile-cache + - python3-fedmsg-core + - python3-fedmsg-meta-fedora-infrastructure + - python3-flask + - python3-flask-oidc + - python3-html5lib + - python3-munch + - python3-pytz + - python3-sqlalchemy + - python3-markdown + - python3-pkgwat-api + - python3-six + - python3-pygments + - python3-pygments-markdown-lexer + - python3-retask # Create directory structure @@ -84,6 +63,7 @@ requirements: "{{ hubs_code_dir }}/requirements.txt" virtualenv: "{{ hubs_venv_dir}}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 - name: Install Fedora Hubs test-requirements.txt into hubs virtualenv become_user: "{{ main_user }}" @@ -91,6 +71,7 @@ requirements: "{{ hubs_code_dir }}/test-requirements.txt" virtualenv: "{{ hubs_venv_dir}}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 - name: Install other packages into hubs virtualenv become_user: "{{ main_user }}" @@ -98,6 +79,7 @@ name: "{{ item }}" virtualenv: "{{ hubs_venv_dir }}" virtualenv_site_packages: yes + virtualenv_command: virtualenv-3 with_items: - bleach @@ -105,7 +87,7 @@ become_user: "{{ main_user }}" command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}" args: - creates: "{{ hubs_venv_dir }}/lib/python2.7/site-packages/fedora-hubs.egg-link" + creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link" - name: Set bin file context in the virtualenv become_user: "{{ main_user }}" @@ -144,8 +126,14 @@ creates: "{{ hubs_conf_dir }}/client_secrets.json" +- name: Start and enable the common services + service: name={{ item }} state=started enabled=yes + with_items: + - redis + - postfix + # Set up, create, and populate the database. -- import_tasks: db-{{ hubs_db_type }}.yml +- include_tasks: db-{{ hubs_db_type }}.yml # Set up JavaScript requirements @@ -156,7 +144,7 @@ creates: node_modules chdir: "{{ hubs_code_dir }}/hubs/static/client" -- name: Build JavaScript assests +- name: Build JavaScript assets command: npm run build become_user: "{{ main_user }}" args: @@ -164,51 +152,18 @@ creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js" -- name: Fix permissions if necessary - file: - path: "{{ item }}" - state: directory - owner: "{{ main_user }}" - group: "{{ main_user }}" - recurse: yes - #setype: httpd_sys_content_rw_t - with_items: - - "{{ hubs_base_dir }}" - - "{{ hubs_conf_dir }}" - - "{{ hubs_var_dir }}" - - # Services -- name: Disable the system-wide fedmsg-hub - service: name=fedmsg-hub state=stopped enabled=no - -- name: Install the service files - template: - src: "{{ item }}.service" - dest: /etc/systemd/system/{{ item }}.service +- name: Disable the system-wide fedmsg daemons + service: name={{ item }} state=stopped enabled=no with_items: - - hubs-triage@ - - hubs-worker@ - - hubs-sse - - hubs-fedmsg-hub - register: service_installed - -- name: reload systemd - command: systemctl daemon-reload - when: service_installed|changed - -- name: Start and enable the services - service: name={{ item }} state=started enabled=yes - with_items: - - redis - - hubs-triage@1 - - hubs-triage@2 - - hubs-worker@1 - - hubs-worker@2 - - hubs-sse - - hubs-fedmsg-hub + - fedmsg-hub + - fedmsg-relay -# Webserver -- import_tasks: webserver.yml +# Include mode-specific tasks + +- include_tasks: dev.yml + when: hubs_dev_mode + +- include_tasks: prod.yml when: not hubs_dev_mode diff --git a/roles/hubs/tasks/prod.yml b/roles/hubs/tasks/prod.yml new file mode 100644 index 0000000000..0cc4894eab --- /dev/null +++ b/roles/hubs/tasks/prod.yml @@ -0,0 +1,28 @@ +- name: Install the service files + template: + src: "{{ item }}.service" + dest: /etc/systemd/system/{{ item }}.service + with_items: + - hubs-triage@ + - hubs-worker@ + - hubs-sse + - hubs-fedmsg-hub + - hubs-fedmsg-relay + register: service_installed + +- name: reload systemd + command: systemctl daemon-reload + when: service_installed|changed + +- name: Start and enable the services in prod mode + service: name={{ item }} state=started enabled=yes + with_items: + - hubs-triage@1 + - hubs-triage@2 + - hubs-worker@1 + - hubs-worker@2 + - hubs-sse + - hubs-fedmsg-hub + - hubs-fedmsg-relay + +- include_tasks: webserver.yml diff --git a/roles/hubs/tasks/webserver.yml b/roles/hubs/tasks/webserver.yml index 3db6d340e4..3aea933ed2 100644 --- a/roles/hubs/tasks/webserver.yml +++ b/roles/hubs/tasks/webserver.yml @@ -8,6 +8,17 @@ - libsemanage-python +- name: install python2-certbot-nginx + dnf: name=python2-certbot-nginx state=present + +- name: get the letencrypt cert + command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org + args: + creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem + notify: + - restart nginx + + - name: Gunicorn logging configuration copy: src: logging.ini @@ -37,17 +48,6 @@ - restart nginx -- name: install python2-certbot-nginx - dnf: name=python2-certbot-nginx state=present - -- name: get the letencrypt cert - command: certbot certonly -n --nginx -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org - args: - creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem - notify: - - restart nginx - - - name: Nginx proxy configuration copy: src: "{{ item }}" diff --git a/roles/hubs/templates/bashrc b/roles/hubs/templates/bashrc index 89027290a4..c1f32e2910 100644 --- a/roles/hubs/templates/bashrc +++ b/roles/hubs/templates/bashrc @@ -13,6 +13,10 @@ fi # by defining a variable with name __help containing the help text +# Honcho has issues outputing UTF-8 in Vagrant SSH +# https://github.com/nickstenning/honcho/issues/51 +export PYTHONIOENCODING=utf-8 + export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py export FLASK_APP={{ hubs_code_dir }}/hubs/app.py @@ -23,14 +27,9 @@ workon() { cd {{ hubs_code_dir }} } -hup() { - source {{ hubs_venv_dir }}/bin/activate - pushd {{ hubs_code_dir }} - FLASK_DEBUG=1 flask run --host 0.0.0.0 --port 5000 -} +alias hup="pushd ~ ; honcho start ; popd" hreset() { - source {{ hubs_venv_dir }}/bin/activate {% if hubs_db_type == "postgresql" %} sudo -u postgres dropdb hubs sudo -u postgres createdb -O hubs hubs @@ -39,7 +38,6 @@ hreset() { {% endif %} rm {{ hubs_var_dir }}/cache.db pushd {{ hubs_code_dir }} - python populate.py + {{ hubs_venv_dir }}/bin/python populate.py popd - deactivate } diff --git a/roles/hubs/templates/fedmsg_config b/roles/hubs/templates/fedmsg_config index 064401878d..1ab54b00a5 100644 --- a/roles/hubs/templates/fedmsg_config +++ b/roles/hubs/templates/fedmsg_config @@ -20,10 +20,13 @@ config = { 'hubs.consumer.enabled': True, 'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue', - # FAS + # Use fedmsg-relay to publish messages + 'active': True, + + # FAS credentials 'fas_credentials': { - 'username': '{{ fedoraDummyUser }}', - 'password': '{{ fedoraDummyUserPassword }}', + 'username': '{{ hubs_fas_username }}', + 'password': '{{ hubs_fas_password }}', }, } diff --git a/roles/hubs/templates/honcho-env b/roles/hubs/templates/honcho-env new file mode 100644 index 0000000000..352c551383 --- /dev/null +++ b/roles/hubs/templates/honcho-env @@ -0,0 +1,3 @@ +FLASK_DEBUG=1 +FLASK_APP={{ hubs_code_dir }}/hubs/app.py +HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py diff --git a/roles/hubs/templates/honcho-procfile b/roles/hubs/templates/honcho-procfile new file mode 100644 index 0000000000..937ef33b6f --- /dev/null +++ b/roles/hubs/templates/honcho-procfile @@ -0,0 +1,7 @@ +web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask-3 run --host 0.0.0.0 --port 5000 +triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage +worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker +sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac +fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub +fedmsg_relay: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay +js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev diff --git a/roles/hubs/templates/hubs-fedmsg-relay.service b/roles/hubs/templates/hubs-fedmsg-relay.service new file mode 100644 index 0000000000..d8fe0ca491 --- /dev/null +++ b/roles/hubs/templates/hubs-fedmsg-relay.service @@ -0,0 +1,14 @@ +[Unit] +Description=Hubs-specific fedmsg processing relay +After=network.target +Documentation=https://fedmsg.readthedocs.org/ + +[Service] +ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay +Type=simple +User=fedmsg +Group=fedmsg +Restart=on-failure + +[Install] +WantedBy=multi-user.target