Hubs: update playbook to use RPM

This commit is contained in:
Aurélien Bompard 2018-02-01 09:33:45 +00:00
parent 9d1075e993
commit ee65d47e58
23 changed files with 207 additions and 352 deletions

View file

@ -33,33 +33,18 @@
- import_tasks: "{{ tasks_path }}/yumrepos.yml"
- dnf: name={{item}} state=present
with_items:
- git
# for certbot
- httpd
- name: create the code directory
file: dest=/srv/hubs state=directory owner=fedora group=fedora
- name: git clone the code
git: repo=https://pagure.io/fedora-hubs.git
dest=/srv/hubs/fedora-hubs
version=develop
become: true
become_user: fedora
#ignore_errors: true
roles:
- basessh
- role: hubs
main_user: fedora
main_user: hubs
hubs_url_hostname: "{{ ansible_fqdn }}"
hubs_secret_key: demotestinghubsmachine
hubs_db_type: postgresql
hubs_dev_mode: false
hubs_conf_dir: /etc/fedora-hubs
hubs_var_dir: /var/lib/fedora-hubs
hubs_ssl_cert: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem
hubs_ssl_key: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
hubs_fas_username: "{{ fedoraDummyUser }}"

View file

@ -4,7 +4,6 @@ hubs_secret_key: changeme
hubs_base_dir: "/srv/hubs"
hubs_code_dir: "{{ hubs_base_dir }}/fedora-hubs"
hubs_conf_dir: "{{ hubs_base_dir }}/config"
hubs_venv_dir: "{{ hubs_base_dir }}/venv"
hubs_var_dir: "{{ hubs_base_dir }}/var"
hubs_db_type: sqlite
hubs_db_password: changeme
@ -12,5 +11,6 @@ hubs_url_hostname: "{{ ansible_fqdn }}"
hubs_url: http{% if not hubs_dev_mode %}s{% endif %}://{{ hubs_url_hostname }}{% if hubs_dev_mode %}:5000{% endif %}
hubs_ssl_cert: /etc/pki/tls/certs/{{ hubs_url_hostname }}.crt
hubs_ssl_key: /etc/pki/tls/private/{{ hubs_url_hostname }}.key
hubs_fas_username: changeme
hubs_fas_password: changeme
hubs_fas_username: null
hubs_fas_password: null
hubs_oidc_url: iddev.fedorainfracloud.org

View file

@ -1,26 +0,0 @@
[loggers]
keys=root
[handlers]
keys=console
[formatters]
keys=simple,minimal
[logger_root]
level=DEBUG
handlers=console
[handler_console]
class=StreamHandler
level=DEBUG
formatter=minimal
args=(sys.stdout,)
[formatter_simple]
format=[%(asctime)s][%(process)d][%(levelname)s] (%(name)s) %(message)s
datefmt=%H:%M:%S
[formatter_minimal]
format=[%(levelname)s] (%(name)s) %(message)s
datefmt=%H:%M:%S

View file

@ -1,29 +1,24 @@
- name: restart postgresql
service: name=postgresql state=restarted
- name: restart the hubs-specific fedmsg-hub
service: name=hubs-fedmsg-hub state=restarted
listen: "hubs configuration change"
when: not hubs_dev_mode
- name: restart hubs triage
service: name=hubs-triage@* state=restarted
service: name=fedora-hubs-triage@* state=restarted
listen: "hubs configuration change"
when: not hubs_dev_mode
- name: restart hubs workers
service: name=hubs-worker@* state=restarted
service: name=fedora-hubs-worker@* state=restarted
listen: "hubs configuration change"
when: not hubs_dev_mode
- name: restart hubs SSE server
service: name=hubs-sse state=restarted
service: name=fedora-hubs-sse state=restarted
listen: "hubs configuration change"
when: not hubs_dev_mode
# Webserver
- name: restart hubs webapp
service: name=hubs-webapp state=restarted
service: name=fedora-hubs-webapp state=restarted
listen: "hubs configuration change"
when: not hubs_dev_mode

View file

@ -5,7 +5,7 @@
with_items:
- postgresql-server
- python3-psycopg2
# For the ansible modules
# For the ansible module
- python-psycopg2
- name: Set up postgresql database
@ -41,21 +41,12 @@
become: true
become_user: postgres
- name: Ease local access to the database
copy:
content: "*:*:hubs:hubs:{{ hubs_db_password }}"
dest: /home/{{ main_user }}/.pgpass
mode: 600
owner: "{{ main_user }}"
group: "{{ main_user }}"
- name: Populate the Fedora Hubs database
command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py"
command: "python3 {{ hubs_code_dir }}/populate.py"
args:
chdir: "{{ hubs_code_dir }}"
environment:
HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py"
HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs.py"
become: true
become_user: "{{ main_user }}"
when: db_creation|succeeded and db_creation|changed
when: db_creation|succeeded and db_creation|changed and hubs_dev_mode

View file

@ -1,9 +1,9 @@
- name: Create and populate the Fedora Hubs database
command: "{{ hubs_venv_dir }}/bin/python {{ hubs_code_dir }}/populate.py"
command: "python3 {{ hubs_code_dir }}/populate.py"
args:
creates: "{{ hubs_var_dir }}/hubs.db"
chdir: "{{ hubs_code_dir }}"
environment:
HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs_config.py"
HUBS_CONFIG: "{{ hubs_conf_dir }}/hubs.py"
become: true
become_user: "{{ main_user }}"

View file

@ -1,17 +1,69 @@
- name: Install Fedora Hubs development packages
# Set up the Python development environment
- name: Install Fedora Hubs requirements.txt into hubs virtualenv
pip:
requirements: "{{ hubs_code_dir }}/requirements.txt"
executable: pip3
- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv
pip:
requirements: "{{ hubs_code_dir }}/test-requirements.txt"
executable: pip3
- name: Install other packages into hubs virtualenv
pip:
name: "{{ item }}"
executable: pip3
with_items:
- bleach
- name: Install Fedora Hubs into the virtualenv
command: "pip3 install -e {{ hubs_code_dir }}"
args:
creates: "/usr/lib/python3.6/site-packages/fedora-hubs.egg-link"
# Set up JavaScript requirements
- name: Install npm packages
command: npm install
become: true
become_user: "{{ main_user }}"
args:
creates: node_modules
chdir: "{{ hubs_code_dir }}/hubs/static/client"
- name: Build JavaScript assets
command: npm run build
become: true
become_user: "{{ main_user }}"
args:
chdir: "{{ hubs_code_dir }}/hubs/static/client"
creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js"
# Development tools
- name: Install helpful development packages
dnf: name={{ item }} state=present
with_items:
- git
- vim-enhanced
- name: Install Fedora Hubs development tools
dnf: name={{ item }} state=present
with_items:
- gcc
- gcc-c++
- libffi-devel
- openssl-devel
- python-sphinx
- python2-devel
- python3-devel
- python3-honcho
- python3-tox
- redhat-rpm-config
- sqlite-devel
- name: Ease local access to the database
copy:
content: "*:*:hubs:hubs:{{ hubs_db_password }}"
dest: /home/{{ main_user }}/.pgpass
mode: 600
owner: "{{ main_user }}"
group: "{{ main_user }}"
when: hubs_db_type == "postgresql"
- name: Install a custom bashrc
template: src=bashrc dest=/home/{{ main_user }}/.bashrc

View file

@ -0,0 +1,64 @@
- name: Install Fedora Hubs development packages
dnf: name={{ item }} state=present
with_items:
- gcc
- gcc-c++
- libffi-devel
- openssl-devel
- python-sphinx
- python2-devel
- python3-devel
- python3-virtualenv
- python3-flask-oidc
- python3-moksha-common
- redhat-rpm-config
- sqlite-devel
- npm
- fedmsg-hub
- name: Install the distribution versions of requirements.txt
dnf: name={{ item }} state=present
with_items:
- python3-alembic
- python3-arrow
- python3-beautifulsoup4
- python3-bleach
- python3-blinker
- python3-dateutil
- python3-decorator
- python3-dogpile-cache
- python3-fedmsg
- python3-fedmsg-meta-fedora-infrastructure
- python3-fedora
- python3-flask
- python3-flask-oidc
- python3-html5lib
- python3-humanize
- python3-iso3166
- python3-markdown
- python3-munch
- python3-pkgwat-api
- python3-pygments
- python3-pygments-markdown-lexer
- python3-pymongo
- python3-pytz
- python3-redis
- python3-requests
- python3-retask
- python3-six
- python3-sqlalchemy
- python3-twisted
- name: Create the directory structure
file:
path: "{{ item.path }}"
state: directory
owner: "{{ main_user }}"
group: "{{ main_user }}"
mode: "{{ item.mode }}"
#setype: httpd_sys_content_rw_t
with_items:
- {path: "{{ hubs_base_dir }}", mode: 755}
- {path: "{{ hubs_conf_dir }}", mode: 750}
- {path: "{{ hubs_var_dir }}", mode: 750}

View file

@ -1,139 +1,57 @@
---
- name: Install helpful development packages
dnf: name={{ item }} state=present
with_items:
- git
- vim-enhanced
- name: Install external dependencies
dnf: name={{ item }} state=present
with_items:
- npm
- redis
- fedmsg-hub
- fedmsg-relay
- python3-virtualenv
- python3-flask-oidc
- python3-moksha-common
- python3-fedmsg
- postfix
- name: Install the distribution versions of requirements.txt
dnf: name={{ item }} state=present
with_items:
- python3-alembic
- python3-arrow
- python3-bleach
- python3-decorator
- python3-dogpile-cache
- python3-fedmsg-core
- python3-fedmsg-meta-fedora-infrastructure
- python3-flask
- python3-flask-oidc
- python3-html5lib
- python3-munch
- python3-pytz
- python3-sqlalchemy
- python3-markdown
- python3-pkgwat-api
- python3-six
- python3-pygments
- python3-pygments-markdown-lexer
- python3-retask
- python3-twisted
- include_tasks: dev_deps.yml
when: hubs_dev_mode
# Create directory structure
- include_tasks: prod_deps.yml
when: not hubs_dev_mode
- name: Create the directory structure
file:
path: "{{ item.path }}"
state: directory
owner: "{{ main_user }}"
group: "{{ main_user }}"
mode: "{{ item.mode }}"
#setype: httpd_sys_content_rw_t
with_items:
- {path: "{{ hubs_base_dir }}", mode: 755}
- {path: "{{ hubs_conf_dir }}", mode: 750}
- {path: "{{ hubs_var_dir }}", mode: 750}
# Set up the Python development environment
- name: Install Fedora Hubs requirements.txt into hubs virtualenv
become: true
become_user: "{{ main_user }}"
pip:
requirements: "{{ hubs_code_dir }}/requirements.txt"
virtualenv: "{{ hubs_venv_dir}}"
virtualenv_site_packages: yes
virtualenv_command: virtualenv-3
- name: Install Fedora Hubs test-requirements.txt into hubs virtualenv
become: true
become_user: "{{ main_user }}"
pip:
requirements: "{{ hubs_code_dir }}/test-requirements.txt"
virtualenv: "{{ hubs_venv_dir}}"
virtualenv_site_packages: yes
virtualenv_command: virtualenv-3
- name: Install other packages into hubs virtualenv
become: true
become_user: "{{ main_user }}"
pip:
name: "{{ item }}"
virtualenv: "{{ hubs_venv_dir }}"
virtualenv_site_packages: yes
virtualenv_command: virtualenv-3
with_items:
- bleach
- name: Install Fedora Hubs into the virtualenv
become: true
become_user: "{{ main_user }}"
command: "{{ hubs_venv_dir }}/bin/pip install -e {{ hubs_code_dir }}"
args:
creates: "{{ hubs_venv_dir }}/lib/python3.6/site-packages/fedora-hubs.egg-link"
- name: Set bin file context in the virtualenv
become: true
become_user: "{{ main_user }}"
file:
path: "{{ hubs_venv_dir }}/bin"
state: directory
recurse: true
setype: bin_t
- name: Add a basic Hubs configuration file
template:
src: "{{ item }}"
dest: "{{ hubs_conf_dir }}/hubs_config.py"
dest: "{{ hubs_conf_dir }}/hubs.py"
owner: root
group: "{{ main_user }}"
mode: 0640
with_first_found:
- hubs_config.{{ ansible_hostname }}
- hubs_config
become: true
become_user: "{{ main_user }}"
notify: "hubs configuration change"
- name: Add a basic fedmsg configuration file
template:
src: "{{ item }}"
dest: "/etc/fedmsg.d/hubs_config.py"
dest: "/etc/fedmsg.d/fedora-hubs.py"
with_first_found:
- fedmsg_config.{{ ansible_hostname }}
- fedmsg_config
notify: "hubs configuration change"
- name: Configure application to authenticate with iddev.fedorainfracloud.org
command:
oidc-register
--output-file {{ hubs_conf_dir }}/client_secrets.json
https://iddev.fedorainfracloud.org/ {{ hubs_url }}
become: true
become_user: "{{ main_user }}"
https://{{ hubs_oidc_url }}/ {{ hubs_url }}
args:
creates: "{{ hubs_conf_dir }}/client_secrets.json"
- name: Set permissions on the oidc credentials file
file:
path: "{{ hubs_conf_dir }}/client_secrets.json"
owner: root
group: "{{ main_user }}"
mode: 0640
- name: Start and enable the common services
service: name={{ item }} state=started enabled=yes
@ -145,29 +63,14 @@
- include_tasks: db-{{ hubs_db_type }}.yml
# Set up JavaScript requirements
- name: Install npm packages
command: npm install
become: true
become_user: "{{ main_user }}"
args:
creates: node_modules
chdir: "{{ hubs_code_dir }}/hubs/static/client"
- name: Build JavaScript assets
command: npm run build
become: true
become_user: "{{ main_user }}"
args:
chdir: "{{ hubs_code_dir }}/hubs/static/client"
creates: "{{ hubs_code_dir }}/hubs/static/js/build/common.js"
# Services
- name: Disable the system-wide fedmsg daemons
service: name={{ item }} state=stopped enabled=no
with_items:
# We use honcho in dev mode and fedmsg-hub-3 in prod mode
- fedmsg-hub
# We use honcho in dev mode and fedmsg-relay-3 in prod mode
- fedmsg-relay
# Include mode-specific tasks

View file

@ -1,27 +1,19 @@
- name: Install the service files
- name: Install the service environment file
template:
src: "{{ item }}.service"
dest: /etc/systemd/system/{{ item }}.service
with_items:
- hubs-triage@
- hubs-worker@
- hubs-sse
- hubs-fedmsg-hub
register: service_installed
- name: reload systemd
command: systemctl daemon-reload
when: service_installed|changed
src: env
dest: /etc/sysconfig/fedora-hubs
- name: Start and enable the services in prod mode
service: name={{ item }} state=started enabled=yes
with_items:
- fedmsg-relay
- hubs-triage@1
- hubs-triage@2
- hubs-worker@1
- hubs-worker@2
- hubs-sse
- hubs-fedmsg-hub
- fedmsg-relay-3
- fedmsg-hub-3
- fedora-hubs-triage@1
- fedora-hubs-triage@2
- fedora-hubs-worker@1
- fedora-hubs-worker@2
- fedora-hubs-worker@3
- fedora-hubs-worker@4
- fedora-hubs-sse
- include_tasks: webserver.yml

View file

@ -0,0 +1,2 @@
- name: Install the Fedora Hubs package
dnf: name=fedora-hubs state=present

View file

@ -11,7 +11,7 @@
- name: install python3-certbot-nginx
dnf: name=python3-certbot-nginx state=present
- name: get the letencrypt cert
- name: get the letsencrypt cert
command: certbot certonly -n --standalone --pre-hook "systemctl stop nginx" --post-hook "systemctl start nginx" -d {{ ansible_fqdn }} --agree-tos --email admin@fedoraproject.org
args:
creates: /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
@ -19,20 +19,10 @@
- restart nginx
- name: Gunicorn logging configuration
copy:
src: logging.ini
dest: "{{ hubs_conf_dir }}/logging.ini"
owner: "{{ main_user }}"
group: "{{ main_user }}"
notify:
- restart hubs webapp
- name: Nginx configuration for hubs
template:
src: nginx.conf
dest: /etc/nginx/conf.d/hubs.conf
dest: /etc/nginx/conf.d/fedora-hubs.conf
notify:
- restart nginx
@ -66,22 +56,8 @@
persistent: yes
- name: Install the service files
template:
src: "{{ item }}.service"
dest: /etc/systemd/system/{{ item }}.service
with_items:
- hubs-webapp
register: service_installed
- name: reload systemd
command: systemctl daemon-reload
when: service_installed|changed
- name: Start and enable the services
service: name={{ item }} state=started enabled=yes
with_items:
- hubs-webapp
- fedora-hubs-webapp
- nginx

View file

@ -5,6 +5,9 @@ if [ -f /etc/bashrc ]; then
. /etc/bashrc
fi
alias vi=vim
# Uncomment the following line if you don't like systemctl's auto-paging feature:
# export SYSTEMD_PAGER=
@ -17,13 +20,12 @@ fi
# https://github.com/nickstenning/honcho/issues/51
export PYTHONIOENCODING=utf-8
export HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
export HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py
export FLASK_APP={{ hubs_code_dir }}/hubs/app.py
workon() {
[ "$1" == "hubs" ] || ( echo "No such virtualenv."; exit 1 )
source {{ hubs_venv_dir }}/bin/activate
cd {{ hubs_code_dir }}
}
@ -38,6 +40,6 @@ hreset() {
{% endif %}
rm {{ hubs_var_dir }}/cache.db
pushd {{ hubs_code_dir }}
{{ hubs_venv_dir }}/bin/python populate.py
python3 populate.py
popd
}

2
roles/hubs/templates/env Normal file
View file

@ -0,0 +1,2 @@
HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py
LOGGING_CONFIG={{ hubs_conf_dir }}/logging.ini

View file

@ -16,17 +16,14 @@ config = {
},
},
# Fedmsg hub consumer
'hubs.consumer.enabled': True,
'hubs.redis.triage-queue-name': 'fedora-hubs-triage-queue',
# Use fedmsg-relay to publish messages
'active': True,
{% if hubs_fas_username and hubs_fas_password %}
# FAS credentials
'fas_credentials': {
'username': '{{ hubs_fas_username }}',
'password': '{{ hubs_fas_password }}',
},
}
}
{% endif %}
# Use fedmsg-relay to publish messages
'active': True,
}

View file

@ -1,3 +1,3 @@
FLASK_DEBUG=1
FLASK_APP={{ hubs_code_dir }}/hubs/app.py
HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
HUBS_CONFIG={{ hubs_conf_dir }}/hubs.py

View file

@ -1,7 +1,7 @@
web: {{ hubs_venv_dir }}/bin/python /usr/bin/flask-3 run --host 0.0.0.0 --port 5000
triage: {{ hubs_venv_dir }}/bin/fedora-hubs-triage
worker: {{ hubs_venv_dir }}/bin/fedora-hubs-worker
sse: {{ hubs_venv_dir }}/bin/python /usr/bin/twistd -l - --pidfile= -ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac
fedmsg_hub: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub
fedmsg_relay: {{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-relay
web: /usr/bin/flask-3 run --host 0.0.0.0 --port 5000
triage: fedora-hubs-triage
worker: fedora-hubs-worker
sse: /usr/bin/twistd-3 -l - --pidfile= -n hubs-sse
fedmsg_hub: /usr/bin/fedmsg-hub-3
fedmsg_relay: /usr/bin/fedmsg-relay-3
js_build: cd {{ hubs_code_dir }}/hubs/static/client && npm run dev

View file

@ -1,14 +0,0 @@
[Unit]
Description=Hubs-specific fedmsg processing hub
After=network.target
Documentation=https://fedmsg.readthedocs.org/
[Service]
ExecStart={{ hubs_venv_dir }}/bin/python /usr/bin/fedmsg-hub
Type=simple
User=fedmsg
Group=fedmsg
Restart=on-failure
[Install]
WantedBy=multi-user.target

View file

@ -1,18 +0,0 @@
[Unit]
Description=fedora-hubs SSE server
After=network.target
Documentation=https://pagure.io/fedora-hubs/
[Service]
ExecStart= \
{{ hubs_venv_dir }}/bin/python \
/usr/bin/twistd -l - --pidfile= \
-ny {{ hubs_code_dir }}/hubs/backend/sse_server.tac
Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
Type=simple
User={{ main_user }}
Group={{ main_user }}
Restart=on-failure
[Install]
WantedBy=multi-user.target

View file

@ -1,15 +0,0 @@
[Unit]
Description=fedora-hubs triage worker #%i
After=network.target
Documentation=https://pagure.io/fedora-hubs/
[Service]
ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-triage
Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
Type=simple
User={{ main_user }}
Group={{ main_user }}
Restart=on-failure
[Install]
WantedBy=multi-user.target

View file

@ -1,20 +0,0 @@
[Unit]
Description=fedora-hubs frontend webapp
After=network.target
Documentation=https://pagure.io/fedora-hubs/
[Service]
ExecStart= \
{{ hubs_venv_dir }}/bin/python \
/usr/bin/python3-gunicorn -b 127.0.0.1:8000 --threads 12 \
--log-config {{ hubs_conf_dir }}/logging.ini \
{% if hubs_dev_mode %}--reload{% endif %} \
hubs.app:app
Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
Type=simple
User={{ main_user }}
Group={{ main_user }}
Restart=on-failure
[Install]
WantedBy=multi-user.target

View file

@ -1,15 +0,0 @@
[Unit]
Description=fedora-hubs cache worker #%i
After=network.target
Documentation=https://pagure.io/fedora-hubs/
[Service]
ExecStart={{ hubs_venv_dir }}/bin/fedora-hubs-worker
Environment=HUBS_CONFIG={{ hubs_conf_dir }}/hubs_config.py
Type=simple
User={{ main_user }}
Group={{ main_user }}
Restart=on-failure
[Install]
WantedBy=multi-user.target

View file

@ -34,14 +34,16 @@ server {
include ssl_params;
keepalive_timeout 5;
# path for static files
root {{ hubs_code_dir }}/hubs/static;
location / {
# checks for static file, if not found proxy to app
try_files $uri @proxy_to_app;
}
# path for static files
location /static {
alias /usr/lib/python3.6/site-packages/hubs/static;
}
location /sse/ {
include proxy_params;
proxy_pass http://hubs-sse/;