From 538b09de361fb3c1ed3a6562f03e2986e996c8e3 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Fri, 29 Jan 2016 19:21:15 +0000 Subject: [PATCH] New darkserver playbooks and roles --- playbooks/groups/darkserver-backend.yml | 67 +++++++ playbooks/groups/darkserver-web.yml | 41 +++++ roles/darkserver/backend/files/email.json | 1 + roles/darkserver/backend/tasks/main.yml | 19 ++ .../backend/templates/darkjobworker.conf.j2 | 11 ++ roles/darkserver/web/files/email.json | 1 + roles/darkserver/web/tasks/main.yml | 22 +++ .../web/templates/darkserverweb.conf.j2 | 10 + roles/darkserver/web/templates/settings.py | 172 ++++++++++++++++++ 9 files changed, 344 insertions(+) create mode 100644 playbooks/groups/darkserver-backend.yml create mode 100644 playbooks/groups/darkserver-web.yml create mode 100644 roles/darkserver/backend/files/email.json create mode 100644 roles/darkserver/backend/tasks/main.yml create mode 100644 roles/darkserver/backend/templates/darkjobworker.conf.j2 create mode 100644 roles/darkserver/web/files/email.json create mode 100644 roles/darkserver/web/tasks/main.yml create mode 100644 roles/darkserver/web/templates/darkserverweb.conf.j2 create mode 100644 roles/darkserver/web/templates/settings.py diff --git a/playbooks/groups/darkserver-backend.yml b/playbooks/groups/darkserver-backend.yml new file mode 100644 index 0000000000..4df600f282 --- /dev/null +++ b/playbooks/groups/darkserver-backend.yml @@ -0,0 +1,67 @@ +- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=darkserver:darkserver-stg" + +- name: make the box be real + hosts: darkserver-backend;darkserver-backend-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + pre_tasks: + + - name: "Add koji to hosts file" + lineinfile: dest=/etc/hosts line="10.5.125.36 koji koji.fedoraproject.org" state=present + - yum: name=libsemanage-python state=present + - name: "Set SElinux booleans" + seboolean: name=httpd_can_network_connect_db state=yes persistent=yes + + roles: + - base + - collectd/base + - fas_client + - hosts + - nagios_client + - rsyncd + - sudo + - { role: openvpn/client, when: env != "staging" } + + - role: apache + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: set up fedmsg basics + hosts: darkserver-backend;darkserver-backend-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + roles: + - fedmsg/base + + handlers: + - include: "{{ handlers }}/restart_services.yml" + +- name: dole out the service-specific config + hosts: darkserver-backend;darkserver-backend-stg + user: root + gather_facts: True + + roles: + - fedmsg/hub + - darkserver/backend + - role: collectd/fedmsg-service + process: fedmsg-hub + diff --git a/playbooks/groups/darkserver-web.yml b/playbooks/groups/darkserver-web.yml new file mode 100644 index 0000000000..d483ecec50 --- /dev/null +++ b/playbooks/groups/darkserver-web.yml @@ -0,0 +1,41 @@ +- include: "/srv/web/infra/ansible/playbooks/include/virt-create.yml myhosts=darkserver:darkserver-stg" + +- name: make the box be real + hosts: darkserver-web;darkserver-web-stg + user: root + gather_facts: True + + vars_files: + - /srv/web/infra/ansible/vars/global.yml + - "/srv/private/ansible/vars.yml" + - /srv/web/infra/ansible/vars/{{ ansible_distribution }}.yml + + pre_tasks: + + - name: "Add koji to hosts file" + lineinfile: dest=/etc/hosts line="10.5.125.36 koji koji.fedoraproject.org" state=present + - yum: name=libsemanage-python state=present + - name: "Set SElinux booleans" + seboolean: name=httpd_can_network_connect_db state=yes persistent=yes + + roles: + - base + - collectd/base + - fas_client + - hosts + - nagios_client + - rsyncd + - sudo + - { role: openvpn/client, when: env != "staging" } + + - role: apache + - darkserver/web + + tasks: + - include: "{{ tasks }}/yumrepos.yml" + - include: "{{ tasks }}/2fa_client.yml" + - include: "{{ tasks }}/motd.yml" + + handlers: + - include: "{{ handlers }}/restart_services.yml" + diff --git a/roles/darkserver/backend/files/email.json b/roles/darkserver/backend/files/email.json new file mode 100644 index 0000000000..4e7ed3a55a --- /dev/null +++ b/roles/darkserver/backend/files/email.json @@ -0,0 +1 @@ +"sysadmin-darkserver-members@fedoraproject.org" diff --git a/roles/darkserver/backend/tasks/main.yml b/roles/darkserver/backend/tasks/main.yml new file mode 100644 index 0000000000..1cf2c2c8e3 --- /dev/null +++ b/roles/darkserver/backend/tasks/main.yml @@ -0,0 +1,19 @@ +# +# Setup darkserver packages +# + +- name: setup darkserver + yum: name={{ item }} state=present + with_items: + - darkserver-importer + notify: + - restart fedmsg-hub + +- name: email.json file + copy: src=email.json dest=/etc/darkserver/email.json owner=root group=root mode=0644 + +- name: darkjobworker.conf + template: src=darkjobworker.conf.j2 dest=/etc/darkserver/darkjobworker.conf owner=root group=root mode=640 + notify: reload darkserver + + diff --git a/roles/darkserver/backend/templates/darkjobworker.conf.j2 b/roles/darkserver/backend/templates/darkjobworker.conf.j2 new file mode 100644 index 0000000000..d6644ee141 --- /dev/null +++ b/roles/darkserver/backend/templates/darkjobworker.conf.j2 @@ -0,0 +1,11 @@ +[darkserver] +{% if environment == "staging" %} +host=db01.stg.phx2.fedoraproject.org +password={{ darkserver_stg_writer_password }} +{% else %} +host=db01.phx2.fedoraproject.org +password={{ darkserver_prod_writer_password }} +{% endif %} +database=darkserver +user=darkserver_writer +unique=127.0.0.1 diff --git a/roles/darkserver/web/files/email.json b/roles/darkserver/web/files/email.json new file mode 100644 index 0000000000..4e7ed3a55a --- /dev/null +++ b/roles/darkserver/web/files/email.json @@ -0,0 +1 @@ +"sysadmin-darkserver-members@fedoraproject.org" diff --git a/roles/darkserver/web/tasks/main.yml b/roles/darkserver/web/tasks/main.yml new file mode 100644 index 0000000000..65306f5dad --- /dev/null +++ b/roles/darkserver/web/tasks/main.yml @@ -0,0 +1,22 @@ +# +# Setup darkserver packages +# + +- name: setup darkserver + yum: name={{ item }} state=present + with_items: + - darkserver + +- name: email.json file + copy: src=email.json dest=/etc/darkserver/email.json owner=root group=root mode=0644 + +- name: darkserverweb.conf + template: src=darkserverweb.conf.j2 dest=/etc/darkserver/darkserverweb.conf owner=apache group=apache mode=0640 + notify: reload httpd + +- name: Copy over settings.py + template: src=settings.py + dest=/etc/darkserver/settings.py + mode=0600 owner=apache group=apache + notify: reload httpd + diff --git a/roles/darkserver/web/templates/darkserverweb.conf.j2 b/roles/darkserver/web/templates/darkserverweb.conf.j2 new file mode 100644 index 0000000000..fee2c03a54 --- /dev/null +++ b/roles/darkserver/web/templates/darkserverweb.conf.j2 @@ -0,0 +1,10 @@ +[darkserverweb] +{% if env == "staging" %} +host=db01.stg.phx2.fedoraproject.org +password={{ darkserver_stg_reader_password }} +{% else %} +host=db01.phx2.fedoraproject.org +password={{ darkserver_prod_reader_password }} +{% endif %} +user=darkserver_reader +database=darkserver diff --git a/roles/darkserver/web/templates/settings.py b/roles/darkserver/web/templates/settings.py new file mode 100644 index 0000000000..d971a88f19 --- /dev/null +++ b/roles/darkserver/web/templates/settings.py @@ -0,0 +1,172 @@ +import os +import sys +import logging +import ConfigParser +config = ConfigParser.ConfigParser() +try: + config.read('/etc/darkserver/darkserverweb.conf') +except: # pragma: no cover + pass + +# Django settings for darkserverweb project. +sys.path.append('/usr/lib/python2.7/site-packages/darkserverweb/') + + +DEBUG = False +TEMPLATE_DEBUG = DEBUG + +ALLOWED_HOSTS = [ + {% if env == 'staging' %} + 'darkserver.stg.fedoraproject.org', + {% else %} + 'darkserver.fedoraproject.org', + {% endif %} + '{{ inventory_hostname }}', + '{{ inventory_hostname_short }}', + 'localhost', +] + +ADMINS = ( + # ('Your Name', 'your_email@example.com'), +) + +MANAGERS = ADMINS + +try: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': config.get('darkserverweb','database'), # Or path to database file if using sqlite3. + 'USER': config.get('darkserverweb','user'), # Not used with sqlite3. + 'PASSWORD': config.get('darkserverweb','password') , # Not used with sqlite3. + 'HOST': config.get('darkserverweb','host'), # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. + } + } +except Exception, e: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. + 'NAME': './test.db', # Or path to database file if using sqlite3. + 'USER': '', # Not used with sqlite3. + 'PASSWORD': '', # Not used with sqlite3. + 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. + 'PORT': '', # Set to empty string for default. Not used with sqlite3. + } + } + +# Local time zone for this installation. Choices can be found here: +# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name +# although not all choices may be available on all operating systems. +# On Unix systems, a value of None will cause Django to use the same +# timezone as the operating system. +# If running in a Windows environment this must be set to the same as your +# system time zone. +TIME_ZONE = 'America/Chicago' + +# Language code for this installation. All choices can be found here: +# http://www.i18nguy.com/unicode/language-identifiers.html +LANGUAGE_CODE = 'en-us' + +SITE_ID = 1 + +# If you set this to False, Django will make some optimizations so as not +# to load the internationalization machinery. +#USE_I18N = True + +# If you set this to False, Django will not format dates, numbers and +# calendars according to the current locale +#USE_L10N = True + +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_ROOT = os.path.dirname(__file__) + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '' + +# Absolute path to the directory static files should be collected to. +# Don't put anything in this directory yourself; store your static files +# in apps' "static/" subdirectories and in STATICFILES_DIRS. +# Example: "/home/media/media.lawrence.com/static/" +STATIC_ROOT = '' + +# URL prefix for static files. +# Example: "http://media.lawrence.com/static/" +STATIC_URL = '/static/' + +# URL prefix for admin static files -- CSS, JavaScript and images. +# Make sure to use a trailing slash. +# Examples: "http://foo.com/static/admin/", "/static/admin/". +ADMIN_MEDIA_PREFIX = '/static/admin/' + +# Additional locations of static files +STATICFILES_DIRS = ( + # Put strings here, like "/home/html/static" or "C:/www/django/static". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +# List of finder classes that know how to find static files in +# various locations. +#STATICFILES_FINDERS = ( +# 'django.contrib.staticfiles.finders.FileSystemFinder', +# 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +# 'django.contrib.staticfiles.finders.DefaultStorageFinder', +#) + +# Make this unique, and don't share it with anybody. +SECRET_KEY = 'vop)@ssylsglmcgzxa2)rs3(t6gt-h@#954mr598h16kzzqwi1' + +# List of callables that know how to import templates from various sources. +TEMPLATE_LOADERS = ( + 'django.template.loaders.filesystem.Loader', + 'django.template.loaders.app_directories.Loader', +# 'django.template.loaders.eggs.Loader', +) + +MIDDLEWARE_CLASSES = ( + 'django.middleware.common.CommonMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATE_DIRS = ( + # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". + # Always use forward slashes, even on Windows. + # Don't forget to use absolute paths, not relative paths. +) + +INSTALLED_APPS = ( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'buildid', + #'django.contrib.staticfiles', + # Uncomment the next line to enable the admin: + # 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', +) + +# A sample logging configuration. The only tangible logging +# performed by this configuration is to send an email to +# the site admins on every HTTP 500 error. +# See http://docs.djangoproject.com/en/dev/topics/logging for +# more details on how to customize your logging configuration. +#logging settings +#LOG_FILENAME = 'darkserver.log' +#logging.basicConfig( +# filename=os.path.join('/var', 'log', 'darkserver', LOG_FILENAME), +# level=logging.CRITICAL, +# format='%(pathname)s TIME: %(asctime)s MSG: %(filename)s:%(funcName)s:%(lineno)d %(message)s', +#) +