Add first work on the pkgdb2 playbook and role

playbooks/groups/pkgdb.yml

@@ -0,0 +1,87 @@
+# create a new pkgdb server
+# NOTE: should be used with --limit most of the time
+# NOTE: make sure there is room/space for this server on the vmhost
+# NOTE: most of these vars_path come from group_vars/pkgdb* or from hostvars
+
+- name: make pkgdb
+  hosts: pkgdb-stg;pkgdb
+  user: root
+  gather_facts: False
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  tasks:
+  - include: "{{ tasks }}/virt_instance_create.yml"
+  - include: "{{ tasks }}/accelerate_prep.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+- name: make the box be real
+  hosts: pkgdb-stg;pkgdb
+  user: root
+  gather_facts: True
+  accelerate: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml
+
+  roles:
+  - base
+  - rkhunter
+  - denyhosts
+  - nagios_client
+  - fas_client
+
+  tasks:
+  - include: "{{ tasks }}/hosts.yml"
+  - include: "{{ tasks }}/yumrepos.yml"
+  - include: "{{ tasks }}/2fa_client.yml"
+  - include: "{{ tasks }}/motd.yml"
+  - include: "{{ tasks }}/sudo.yml"
+  - include: "{{ tasks }}/openvpn_client.yml"
+    when: env != "staging"
+  - include: "{{ tasks }}/apache.yml"
+  - include: "{{ tasks }}/mod_wsgi.yml"
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+- name: set up fedmsg on pkgdb
+  hosts: pkgdb-stg;pkgdb
+  user: root
+  gather_facts: True
+  accelerate: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - "{{ vars_path }}/{{ ansible_distribution }}.yml"
+
+  roles:
+  - fedmsg_base
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"
+
+- name: deploy pkgdb itself
+  hosts: pkgdb-stg;pkgdb
+  user: root
+  gather_facts: True
+  accelerate: True
+
+  vars_files:
+   - /srv/web/infra/ansible/vars/global.yml
+   - "{{ private }}/vars.yml"
+   - "{{ vars_path }}/{{ ansible_distribution }}.yml"
+
+  roles:
+  - pkgdb
+
+  handlers:
+  - include: "{{ handlers }}/restart_services.yml"

roles/pkgdb2/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+# Configuration for the pkgdb2 webapp
+
+- name: clean yum metadata
+  command: yum clean all
+  tags:
+  - packages
+
+- name: install needed packages
+  yum: pkg={{ item }} state=installed
+  with_items:
+  - pkgdb2
+  - python-psycopg2
+  - python-openid-cla
+  - python-openid-teams
+  - python-memcached
+  - libsemanage-python
+  tags:
+  - packages
+
+- name: copy sundry pkgdb configuration
+  template: src={{ item.file }}
+            dest={{ item.location }}/{{ item.dest }}
+            owner=apache group=apache mode=0600
+  with_items:
+  - { file: pkgdb2_admin.cfg, location: /etc/pkgdb2, dest: pkgdb2.cfg }
+  - { file: alembic.ini, location: /etc/pkgdb2, dest: alembic.ini }
+  tags:
+  - config
+  notify:
+  - restart apache
+
+- name: create the database scheme
+  command: /usr/bin/python2 /usr/share/pkgdb2/pkgdb2_createdb.py
+  environment:
+      PKGDB2_CONFIG: /etc/pkgdb2/pkgdb2.cfg
+
+- name: Install all the configuration file of pkgdb2
+  template: src={{ item.file }}
+            dest={{ item.location }}/{{ item.file }}
+            owner=apache group=apache mode=0600
+  with_items:
+  - { file: pkgdb2.cfg, location: /etc/pkgdb2 }
+  - { file: pkgdb2.conf, location: /etc/httpd/conf.d }
+  - { file: pkgdb2.wsgi, location: /var/www/, dest: pkgdb2.wsgi }
+  tags:
+  - config
+  notify:
+  - restart apache
+
+- name: set sebooleans so pkgdb2 can talk to the db
+  action: seboolean name=httpd_can_network_connect_db
+                    state=true
+                    persistent=true
+

roles/pkgdb2/templates/alembic.ini

@@ -0,0 +1,51 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = /usr/share/pkgdb/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+#sqlalchemy.url = postgresql://<%= pkgdb_app %>:<%= pkgdb_appPassword %>@db-pkgdb/pkgdb
+sqlalchemy.url = postgresql://{{ pkgdb_db_admin_user }}:{{ pkgdb_db_admin_pass }}@{{ pkgdb_db_host }}/{{ pkgdb_db_name }}
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

roles/pkgdb2/templates/pkgdb2-proxy.conf

@@ -0,0 +1,7 @@
+RewriteEngine On
+
+ProxyPass <%= path %> <%= proxyurl %>/pkgdb
+ProxyPassReverse <%= path %> <%= proxyurl %>/pkgdb
+
+RequestHeader set X-Forwarded-Scheme https early
+

roles/pkgdb2/templates/pkgdb2.cfg

@@ -0,0 +1,74 @@
+# Beware that the quotes around the values are mandatory
+
+### Secret key for the Flask application
+SECRET_KEY='{{ pkgdb2_secret_key }}'
+
+### url to the database server:
+#DB_URL=mysql://user:pass@host/db_name
+#DB_URL=postgres://user:pass@host/db_name
+DB_URL='postgresql://{{ pkgdb2_db_user }}:{{ pkgdb2_db_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}'
+
+### the number of items (packages, packagers..) to display on the search
+### pages
+ITEMS_PER_PAGE = 50
+
+
+### List the ACL which are automatically approved (don't need reviewing)
+AUTO_APPROVE = ['watchcommits', 'watchbugzilla']
+
+#### FAS group for the pkgdb admins
+ADMIN_GROUP = ['sysadmin-main', 'sysadmin-cvs']
+
+### The default backend for dogpile
+### Options are listed at:
+### http://dogpilecache.readthedocs.org/en/latest/api.html  (backend section)
+PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached'
+PKGDB2_CACHE_KWARGS = {
+    'arguments': {
+        'url': "127.0.0.1:11211",
+    }
+}
+
+
+### Bugzilla information
+
+## Upon changes in pkgdb, update bugzilla
+PKGDB2_BUGZILLA_NOTIFICATION = False
+## URL to the bugzilla instance to update
+PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com'
+## name of the user the pkgdb application can log in to bugzilla with
+PKGDB2_BUGZILLA_USER = None
+## password of the user the pkgdb application can log in to bugzilla with
+PKGDB2_BUGZILLA_PASSWORD = None
+
+
+### FAS information
+
+## URL to the FAS instance to query
+PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts'
+## name of the user the pkgdb application can log in to FAS with
+PKGDB2_FAS_USER = '{{ pkgdb_fas_user }}'
+## password of the user the pkgdb application can log in to FAS with
+PKGDB2_FAS_PASSWORD = '{{ pkgdb_fas_password }}'
+
+
+### pkgdb notifications
+
+## Pkgdb broadcasts its notifications via fedmsg
+PKGDB2_FEDMSG_NOTIFICATION = True
+## Pkgdb sends its notifications by email
+PKGDB2_EMAIL_NOTIFICATION = False
+## Template to build the email address pkgdb sends its notifications to
+PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org'
+## The From address email notifications are sent with
+PKGDB2_EMAIL_FROM = 'nobody@fedoraproject.org'
+## The SMTP server to use to send email notifications
+PKGDB2_EMAIL_SMTP_SERVER = 'localhost'
+
+
+### Email stacktrace
+
+## pkgdb sends email when it faces an exception (trying to add an existing
+## package or something alike. These emails are sent to the address set
+## here:
+MAIL_ADMIN = 'pingou@fedoraproject.org'

roles/pkgdb2/templates/pkgdb2.conf

@@ -0,0 +1,23 @@
+
+Alias /pkgdb/static /usr/lib/python2.6/site-packages/pkgdb2/static/default/
+
+WSGIDaemonProcess pkgdb user=apache maximum-requests=1000 display-name=pkgdb processes=2 threads=1
+WSGISocketPrefix run/wsgi
+WSGIRestrictStdout On
+WSGIRestrictSignal Off
+WSGIPythonOptimize 1
+
+WSGIScriptAlias /pkgdb /var/www/pkgdb2.wsgi
+
+<Location />
+    WSGIProcessGroup pkgdb
+    <IfModule mod_authz_core.c>
+        # Apache 2.4
+        Require all granted
+    </IfModule>
+    <IfModule !mod_authz_core.c>
+         # Apache 2.2
+        Order deny,allow
+        Allow from all
+    </IfModule>
+</Location>

roles/pkgdb2/templates/pkgdb2.wsgi

@@ -0,0 +1,22 @@
+#-*- coding: UTF-8 -*-
+
+# The three lines below are required to run on EL6 as EL6 has
+# two possible version of python-sqlalchemy and python-jinja2
+# These lines make sure the application uses the correct version.
+import __main__
+__main__.__requires__ = ['SQLAlchemy >= 0.7', 'jinja2 >= 2.4']
+import pkg_resources
+
+import os
+## Set the environment variable pointing to the configuration file
+os.environ['PKGDB2_CONFIG'] = '/etc/pkgdb2/pkgdb2.cfg'
+
+## The following is only needed if you did not install pkgdb
+## as a python module (for example if you run it from a git clone).
+#import sys
+#sys.path.insert(0, '/path/to/pkgdb/')
+
+
+## The most import line to make the wsgi working
+from pkgdb2 import APP as application
+

roles/pkgdb2/templates/pkgdb2_admin.cfg

@@ -0,0 +1,74 @@
+# Beware that the quotes around the values are mandatory
+
+### Secret key for the Flask application
+SECRET_KEY='{{ pkgdb2_secret_key }}'
+
+### url to the database server:
+#DB_URL=mysql://user:pass@host/db_name
+#DB_URL=postgres://user:pass@host/db_name
+DB_URL='postgresql://{{ pkgdb2_db_admin_user }}:{{ pkgdb2_db_admin_pass }}@{{ pkgdb2_db_host }}/{{ pkgdb2_db_name }}'
+
+### the number of items (packages, packagers..) to display on the search
+### pages
+ITEMS_PER_PAGE = 50
+
+
+### List the ACL which are automatically approved (don't need reviewing)
+AUTO_APPROVE = ['watchcommits', 'watchbugzilla']
+
+#### FAS group for the pkgdb admins
+ADMIN_GROUP = ['sysadmin-main', 'sysadmin-cvs']
+
+### The default backend for dogpile
+### Options are listed at:
+### http://dogpilecache.readthedocs.org/en/latest/api.html  (backend section)
+PKGDB2_CACHE_BACKEND = 'dogpile.cache.memcached'
+PKGDB2_CACHE_KWARGS = {
+    'arguments': {
+        'url': "127.0.0.1:11211",
+    }
+}
+
+
+### Bugzilla information
+
+## Upon changes in pkgdb, update bugzilla
+PKGDB2_BUGZILLA_NOTIFICATION = False
+## URL to the bugzilla instance to update
+PKGDB2_BUGZILLA_URL = 'https://bugzilla.redhat.com'
+## name of the user the pkgdb application can log in to bugzilla with
+PKGDB2_BUGZILLA_USER = None
+## password of the user the pkgdb application can log in to bugzilla with
+PKGDB2_BUGZILLA_PASSWORD = None
+
+
+### FAS information
+
+## URL to the FAS instance to query
+PKGDB2_FAS_URL = 'https://admin.fedoraproject.org/accounts'
+## name of the user the pkgdb application can log in to FAS with
+PKGDB2_FAS_USER = '{{ pkgdb_fas_user }}'
+## password of the user the pkgdb application can log in to FAS with
+PKGDB2_FAS_PASSWORD = '{{ pkgdb_fas_password }}'
+
+
+### pkgdb notifications
+
+## Pkgdb broadcasts its notifications via fedmsg
+PKGDB2_FEDMSG_NOTIFICATION = True
+## Pkgdb sends its notifications by email
+PKGDB2_EMAIL_NOTIFICATION = False
+## Template to build the email address pkgdb sends its notifications to
+PKGDB2_EMAIL_TO = '{pkg_name}-owner@fedoraproject.org'
+## The From address email notifications are sent with
+PKGDB2_EMAIL_FROM = 'nobody@fedoraproject.org'
+## The SMTP server to use to send email notifications
+PKGDB2_EMAIL_SMTP_SERVER = 'localhost'
+
+
+### Email stacktrace
+
+## pkgdb sends email when it faces an exception (trying to add an existing
+## package or something alike. These emails are sent to the address set
+## here:
+MAIL_ADMIN = 'pingou@fedoraproject.org'