diff --git a/playbooks/groups/pkgdb.yml b/playbooks/groups/pkgdb.yml new file mode 100644 index 0000000000..0f797f523f --- /dev/null +++ b/playbooks/groups/pkgdb.yml @@ -0,0 +1,87 @@ +# create a new pkgdb server +# NOTE: should be used with --limit most of the time +# NOTE: make sure there is room/space for this server on the vmhost +# NOTE: most of these vars_path come from group_vars/pkgdb* or from hostvars + +- name: make pkgdb + hosts: pkgdb-stg;pkgdb + user: root + gather_facts: False + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + tasks: + - include: "{{ tasks }}/virt_instance_create.yml" + - include: "{{ tasks }}/accelerate_prep.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: make the box be real + hosts: pkgdb-stg;pkgdb + user: root + gather_facts: True + accelerate: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - base + - rkhunter + - denyhosts + - nagios_client + - fas_client + + tasks: + - include: "{{ tasks }}/hosts.yml" + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + - include: "{{ tasks }}/sudo.yml" + - include: "{{ tasks }}/openvpn_client.yml" + when: env != "staging" + - include: "{{ tasks }}/apache.yml" + - include: "{{ tasks }}/mod_wsgi.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: set up fedmsg on pkgdb + hosts: pkgdb-stg;pkgdb + user: root + gather_facts: True + accelerate: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - "{{ vars_path }}/{{ ansible_distribution }}.yml" + + roles: + - fedmsg_base + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: deploy pkgdb itself + hosts: pkgdb-stg;pkgdb + user: root + gather_facts: True + accelerate: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "{{ private }}/vars.yml" + - "{{ vars_path }}/{{ ansible_distribution }}.yml" + + roles: + - pkgdb + + handlers: + - include: "{{ handlers }}/restart_services.yml" diff --git a/roles/pkgdb2/tasks/main.yml b/roles/pkgdb2/tasks/main.yml new file mode 100644 index 0000000000..248829c67d --- /dev/null +++ b/roles/pkgdb2/tasks/main.yml @@ -0,0 +1,55 @@ +--- +# Configuration for the pkgdb2 webapp + +- name: clean yum metadata + command: yum clean all + tags: + - packages + +- name: install needed packages + yum: pkg={{ item }} state=installed + with_items: + - pkgdb2 + - python-psycopg2 + - python-openid-cla + - python-openid-teams + - python-memcached + - libsemanage-python + tags: + - packages + +- name: copy sundry pkgdb configuration + template: src={{ item.file }} + dest={{ item.location }}/{{ item.dest }} + owner=apache group=apache mode=0600 + with_items: + - { file: pkgdb2_admin.cfg, location: /etc/pkgdb2, dest: pkgdb2.cfg } + - { file: alembic.ini, location: /etc/pkgdb2, dest: alembic.ini } + tags: + - config + notify: + - restart apache + +- name: create the database scheme + command: /usr/bin/python2 /usr/share/pkgdb2/pkgdb2_createdb.py + environment: + PKGDB2_CONFIG: /etc/pkgdb2/pkgdb2.cfg + +- name: Install all the configuration file of pkgdb2 + template: src={{ item.file }} + dest={{ item.location }}/{{ item.file }} + owner=apache group=apache mode=0600 + with_items: + - { file: pkgdb2.cfg, location: /etc/pkgdb2 } + - { file: pkgdb2.conf, location: /etc/httpd/conf.d } + - { file: pkgdb2.wsgi, location: /var/www/, dest: pkgdb2.wsgi } + tags: + - config + notify: + - restart apache + +- name: set sebooleans so pkgdb2 can talk to the db + action: seboolean name=httpd_can_network_connect_db + state=true + persistent=true + diff --git a/roles/pkgdb2/templates/alembic.ini b/roles/pkgdb2/templates/alembic.ini new file mode 100644 index 0000000000..119ef6366d --- /dev/null +++ b/roles/pkgdb2/templates/alembic.ini @@ -0,0 +1,51 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = /usr/share/pkgdb/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +#sqlalchemy.url = postgresql://<%= pkgdb_app %>:<%= pkgdb_appPassword %>@db-pkgdb/pkgdb +sqlalchemy.url = postgresql://{{ pkgdb_db_admin_user }}:{{ pkgdb_db_admin_pass }}@{{ pkgdb_db_host }}/{{ pkgdb_db_name }} + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/roles/pkgdb2/templates/pkgdb2-proxy.conf b/roles/pkgdb2/templates/pkgdb2-proxy.conf new file mode 100644 index 0000000000..3e1449f3ac --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb2-proxy.conf @@ -0,0 +1,7 @@ +RewriteEngine On + +ProxyPass <%= path %> <%= proxyurl %>/pkgdb +ProxyPassReverse <%= path %> <%= proxyurl %>/pkgdb + +RequestHeader set X-Forwarded-Scheme https early + diff --git a/roles/pkgdb2/templates/pkgdb2.cfg b/roles/pkgdb2/templates/pkgdb2.cfg new file mode 100644 index 0000000000..38d50d71a5 --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb2.cfg @@ -0,0 +1,74 @@ +# Beware that the quotes around the values are mandatory + +### Secret key for the Flask application +SECRET_KEY='{{ pkgdb2_secret_key }}' + +### url to the database server: +#DB_URL=mysql://user:pass@host/db_name +#DB_URL=postgres://user:pass@host/db_name +DB_URL='postgresql://{{ pkgdb2_db_user }}:{{ pkgdb2_db_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}' + +### the number of items (packages, packagers..) to display on the search +### pages +ITEMS_PER_PAGE = 50 + + +### List the ACL which are automatically approved (don't need reviewing) +AUTO_APPROVE = ['watchcommits', 'watchbugzilla'] + +#### FAS group for the pkgdb admins +ADMIN_GROUP = ['sysadmin-main', 'sysadmin-cvs'] + +### The default backend for dogpile +### Options are listed at: +### http://dogpilecache.readthedocs.org/en/latest/api.html (backend section) +PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached' +PKGDB2_CACHE_KWARGS = { + 'arguments': { + 'url': "127.0.0.1:11211", + } +} + + +### Bugzilla information + +## Upon changes in pkgdb, update bugzilla +PKGDB2_BUGZILLA_NOTIFICATION = False +## URL to the bugzilla instance to update +PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com' +## name of the user the pkgdb application can log in to bugzilla with +PKGDB2_BUGZILLA_USER = None +## password of the user the pkgdb application can log in to bugzilla with +PKGDB2_BUGZILLA_PASSWORD = None + + +### FAS information + +## URL to the FAS instance to query +PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts' +## name of the user the pkgdb application can log in to FAS with +PKGDB2_FAS_USER = '{{ pkgdb_fas_user }}' +## password of the user the pkgdb application can log in to FAS with +PKGDB2_FAS_PASSWORD = '{{ pkgdb_fas_password }}' + + +### pkgdb notifications + +## Pkgdb broadcasts its notifications via fedmsg +PKGDB2_FEDMSG_NOTIFICATION = True +## Pkgdb sends its notifications by email +PKGDB2_EMAIL_NOTIFICATION = False +## Template to build the email address pkgdb sends its notifications to +PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org' +## The From address email notifications are sent with +PKGDB2_EMAIL_FROM = 'nobody@fedoraproject.org' +## The SMTP server to use to send email notifications +PKGDB2_EMAIL_SMTP_SERVER = 'localhost' + + +### Email stacktrace + +## pkgdb sends email when it faces an exception (trying to add an existing +## package or something alike. These emails are sent to the address set +## here: +MAIL_ADMIN = 'pingou@fedoraproject.org' diff --git a/roles/pkgdb2/templates/pkgdb2.conf b/roles/pkgdb2/templates/pkgdb2.conf new file mode 100644 index 0000000000..221269d2f8 --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb2.conf @@ -0,0 +1,23 @@ + +Alias /pkgdb/static /usr/lib/python2.6/site-packages/pkgdb2/static/default/ + +WSGIDaemonProcess pkgdb user=apache maximum-requests=1000 display-name=pkgdb processes=2 threads=1 +WSGISocketPrefix run/wsgi +WSGIRestrictStdout On +WSGIRestrictSignal Off +WSGIPythonOptimize 1 + +WSGIScriptAlias /pkgdb /var/www/pkgdb2.wsgi + + + WSGIProcessGroup pkgdb + + # Apache 2.4 + Require all granted + + + # Apache 2.2 + Order deny,allow + Allow from all + + diff --git a/roles/pkgdb2/templates/pkgdb2.wsgi b/roles/pkgdb2/templates/pkgdb2.wsgi new file mode 100644 index 0000000000..f485c5922f --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb2.wsgi @@ -0,0 +1,22 @@ +#-*- coding: UTF-8 -*- + +# The three lines below are required to run on EL6 as EL6 has +# two possible version of python-sqlalchemy and python-jinja2 +# These lines make sure the application uses the correct version. +import __main__ +__main__.__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4'] +import pkg_resources + +import os +## Set the environment variable pointing to the configuration file +os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg' + +## The following is only needed if you did not install pkgdb +## as a python module (for example if you run it from a git clone). +#import sys +#sys.path.insert(0, '/path/to/pkgdb/') + + +## The most import line to make the wsgi working +from pkgdb2 import APP as application + diff --git a/roles/pkgdb2/templates/pkgdb2_admin.cfg b/roles/pkgdb2/templates/pkgdb2_admin.cfg new file mode 100644 index 0000000000..16b0d69f0d --- /dev/null +++ b/roles/pkgdb2/templates/pkgdb2_admin.cfg @@ -0,0 +1,74 @@ +# Beware that the quotes around the values are mandatory + +### Secret key for the Flask application +SECRET_KEY='{{ pkgdb2_secret_key }}' + +### url to the database server: +#DB_URL=mysql://user:pass@host/db_name +#DB_URL=postgres://user:pass@host/db_name +DB_URL='postgresql://{{ pkgdb2_db_admin_user }}:{{ pkgdb2_db_admin_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}' + +### the number of items (packages, packagers..) to display on the search +### pages +ITEMS_PER_PAGE = 50 + + +### List the ACL which are automatically approved (don't need reviewing) +AUTO_APPROVE = ['watchcommits', 'watchbugzilla'] + +#### FAS group for the pkgdb admins +ADMIN_GROUP = ['sysadmin-main', 'sysadmin-cvs'] + +### The default backend for dogpile +### Options are listed at: +### http://dogpilecache.readthedocs.org/en/latest/api.html (backend section) +PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached' +PKGDB2_CACHE_KWARGS = { + 'arguments': { + 'url': "127.0.0.1:11211", + } +} + + +### Bugzilla information + +## Upon changes in pkgdb, update bugzilla +PKGDB2_BUGZILLA_NOTIFICATION = False +## URL to the bugzilla instance to update +PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com' +## name of the user the pkgdb application can log in to bugzilla with +PKGDB2_BUGZILLA_USER = None +## password of the user the pkgdb application can log in to bugzilla with +PKGDB2_BUGZILLA_PASSWORD = None + + +### FAS information + +## URL to the FAS instance to query +PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts' +## name of the user the pkgdb application can log in to FAS with +PKGDB2_FAS_USER = '{{ pkgdb_fas_user }}' +## password of the user the pkgdb application can log in to FAS with +PKGDB2_FAS_PASSWORD = '{{ pkgdb_fas_password }}' + + +### pkgdb notifications + +## Pkgdb broadcasts its notifications via fedmsg +PKGDB2_FEDMSG_NOTIFICATION = True +## Pkgdb sends its notifications by email +PKGDB2_EMAIL_NOTIFICATION = False +## Template to build the email address pkgdb sends its notifications to +PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org' +## The From address email notifications are sent with +PKGDB2_EMAIL_FROM = 'nobody@fedoraproject.org' +## The SMTP server to use to send email notifications +PKGDB2_EMAIL_SMTP_SERVER = 'localhost' + + +### Email stacktrace + +## pkgdb sends email when it faces an exception (trying to add an existing +## package or something alike. These emails are sent to the address set +## here: +MAIL_ADMIN = 'pingou@fedoraproject.org'