Merge branch 'master' of /git/ansible

This commit is contained in:
Stephen Smoogen 2016-03-23 22:41:53 +00:00
commit 34e0c1941b
14 changed files with 300 additions and 100 deletions

View file

@ -0,0 +1,163 @@
# -*- coding: utf-8 -*-
import fedmsg.consumers
import koji
from autocloud.utils import get_image_url, produce_jobs, get_image_name
import autocloud
import logging
log = logging.getLogger("fedmsg")
DEBUG = autocloud.DEBUG
class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer):
if DEBUG:
topic = [
'org.fedoraproject.dev.__main__.buildsys.build.state.change',
'org.fedoraproject.dev.__main__.buildsys.task.state.change',
]
else:
topic = [
'org.fedoraproject.prod.buildsys.build.state.change',
'org.fedoraproject.prod.buildsys.task.state.change',
]
config_key = 'autocloud.consumer.enabled'
def __init__(self, *args, **kwargs):
super(AutoCloudConsumer, self).__init__(*args, **kwargs)
def _get_tasks(self, builds):
""" Takes a list of koji createImage task IDs and returns dictionary of
build ids and image url corresponding to that build ids"""
if autocloud.VIRTUALBOX:
_supported_images = ('Fedora-Cloud-Base-Vagrant',
'Fedora-Cloud-Atomic-Vagrant',)
else:
_supported_images = ('Fedora-Cloud-Base-Vagrant',
'Fedora-Cloud-Atomic-Vagrant',
'Fedora-Cloud-Atomic', 'Fedora-Cloud-Base',)
for build in builds:
log.info('Got Koji build {0}'.format(build))
# Create a Koji connection to the Fedora Koji instance
koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL)
image_files = [] # list of full URLs of files
if len(builds) == 1:
task_result = koji_session.getTaskResult(builds[0])
name = task_result.get('name')
#TODO: Change to get the release information from PDC instead
# of koji once it is set up
release = task_result.get('version')
if name in _supported_images:
task_relpath = koji.pathinfo.taskrelpath(int(builds[0]))
url = get_image_url(task_result.get('files'), task_relpath)
if url:
name = get_image_name(image_name=name)
data = {
'buildid': builds[0],
'image_url': url,
'name': name,
'release': release,
}
image_files.append(data)
elif len(builds) >= 2:
koji_session.multicall = True
for build in builds:
koji_session.getTaskResult(build)
results = koji_session.multiCall()
for result in results:
if not result:
continue
name = result[0].get('name')
if name not in _supported_images:
continue
#TODO: Change to get the release information from PDC instead
# of koji once it is set up
release = result[0].get('version')
task_relpath = koji.pathinfo.taskrelpath(
int(result[0].get('task_id')))
url = get_image_url(result[0].get('files'), task_relpath)
if url:
name = get_image_name(image_name=name)
data = {
'buildid': result[0]['task_id'],
'image_url': url,
'name': name,
'release': release,
}
image_files.append(data)
return image_files
def consume(self, msg):
""" This is called when we receive a message matching the topic. """
if msg['topic'].endswith('.buildsys.task.state.change'):
# Do the thing you've always done... this will go away soon.
# releng is transitioning away from it.
self._consume_scratch_task(msg)
elif msg['topic'].endswith('.buildsys.build.state.change'):
# Do the new thing we need to do. handle a 'real build' from koji,
# not just a scratch task.
self._consume_real_build(msg)
else:
raise NotImplementedError("Should be impossible to get here...")
def _consume_real_build(self, msg):
builds = list() # These will be the Koji task IDs to upload, if any.
msg = msg['body']['msg']
if msg['owner'] != 'releng':
log.debug("Dropping message. Owned by %r" % msg['owner'])
return
if msg['instance'] != 'primary':
log.info("Dropping message. From %r instance." % msg['instance'])
return
# Don't upload *any* images if one of them fails.
if msg['new'] != 1:
log.info("Dropping message. State is %r" % msg['new'])
return
koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL)
children = koji_session.getTaskChildren(msg['task_id'])
for child in children:
if child["method"] == "createImage":
builds.append(child["id"])
if len(builds) > 0:
produce_jobs(self._get_tasks(builds))
def _consume_scratch_task(self, msg):
builds = list() # These will be the Koji build IDs to upload, if any.
msg_info = msg["body"]["msg"]["info"]
log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id']))
# If the build method is "image", we check to see if the child
# task's method is "createImage".
if msg_info["method"] == "image":
if isinstance(msg_info["children"], list):
for child in msg_info["children"]:
if child["method"] == "createImage":
# We only care about the image if the build
# completed successfully (with state code 2).
if child["state"] == 2:
builds.append(child["id"])
if len(builds) > 0:
produce_jobs(self._get_tasks(builds))

View file

@ -9,7 +9,7 @@ def invert_fedmsg_policy(groups, vars, env):
"""
if env == 'staging':
hosts = groups['staging']
hosts = groups['staging'] + groups['fedmsg-qa-network-stg']
else:
hosts = [h for h in groups['all'] if h not in groups['staging']]

View file

@ -8,7 +8,10 @@ custom_rules: [
# fas01, fas02, and fas03
'-A INPUT -p tcp -m tcp -s 10.5.126.25 --dport 80 -j ACCEPT',
'-A INPUT -p tcp -m tcp -s 10.5.126.26 --dport 80 -j ACCEPT',
'-A INPUT -p tcp -m tcp -s 10.5.126.30 --dport 80 -j ACCEPT'
'-A INPUT -p tcp -m tcp -s 10.5.126.30 --dport 80 -j ACCEPT',
# wiki01, wiki02
'-A INPUT -p tcp -m tcp -s 10.5.126.63 --dport 80 -j ACCEPT',
'-A INPUT -p tcp -m tcp -s 10.5.126.73 --dport 80 -j ACCEPT',
]
fas_client_groups: sysadmin-main

View file

@ -7,6 +7,8 @@ num_cpus: 2
custom_rules: [
# fas01.stg
'-A INPUT -p tcp -m tcp -s 10.5.126.86 --dport 80 -j ACCEPT',
# wiki01.stg
'-A INPUT -p tcp -m tcp -s 10.5.126.60 --dport 80 -j ACCEPT',
]
fas_client_groups: sysadmin-main

View file

@ -797,6 +797,7 @@ zanata2fedmsg01.phx2.fedoraproject.org
# See also:
# - inventory/group_vars/proxies for the iptables custom_rules list
# - roles/fedmsg/base/templates/relay.py.j2
# - filter_plugins/fedmsg.py
[fedmsg-qa-network]
retrace01.qa.fedoraproject.org
retrace02.qa.fedoraproject.org

View file

@ -35,6 +35,11 @@
mnt_dir: '/mnt/fedora_koji'
nfs_src_dir: 'fedora_koji'
when: datacenter == 'staging'
- role: nfs/client
mnt_dir: '/mnt/fedora_koji_prod'
nfs_src_dir: 'fedora_koji'
nfs_mount_opts: 'ro,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3'
when: datacenter == 'staging'
- releng
- fedmsg/base
- sudo

View file

@ -24,7 +24,7 @@
vars:
# this is actually without admin tenant
all_tenants: ['cloudintern', 'cloudsig', 'copr', 'coprdev', 'infrastructure',
'persistent', 'pythonbots', 'qa', 'scratch', 'transient']
'persistent', 'pythonbots', 'qa', 'scratch', 'transient', 'openshift']
vars_files:
- /srv/web/infra/ansible/vars/global.yml
@ -688,6 +688,7 @@
- { name: coprdev, desc: 'Development version of Copr' }
- { name: pythonbots, desc: 'project for python build bot users - twisted, etc' }
- { name: scratch, desc: 'scratch and short term instances' }
- { name: openshift, desc: 'Tenant for openshift deployment' }
##### USERS #####
@ -727,6 +728,7 @@
- { name: roshi, email: 'roshi@fedoraproject.org', tenant: qa, password: "{{roshi_password}}" }
- { name: maxamillion, email: 'maxamillion@fedoraproject.org', tenant: infrastructure, password: "{{maxamillion_password}}" }
- { name: clime, email: 'clime@redhat.com', tenant: copr, password: "{{clime_password}}" }
- { name: misc, email: 'misc@redhat.com', tenant: openshift, password: "{{misc_password}}" }
tags:
- openstack_users
@ -765,6 +767,7 @@
- { username: admin, name: fedora-admin-20130801, tenant: admin, password: "{{ADMIN_PASS}}", public_key: "{{ lookup('file', files + '/fedora-cloud/fedora-admin-20130801.pub') }}" }
- { username: asamalik, name: asamalik, tenant: scratch, password: "{{asamalik_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas asamalik') }}" }
- { username: clime, name: clime, tenant: copr, password: "{{clime_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas clime') }}" }
- { username: misc, name: misc, tenant: openshift, password: "{{misc_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas misc') }}" }
tags:
- openstack_users
@ -907,6 +910,7 @@
- { name: qa, shared: false }
- { name: scratch, shared: false }
- { name: transient, shared: false }
- { name: openshift, shared: false }
- name: Create a subnet for all tenants
neutron_subnet:
login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin"
@ -928,6 +932,7 @@
- { name: qa, cidr: '172.25.112.1/20', gateway: '172.25.112.1' }
- { name: scratch, cidr: '172.25.64.1/20', gateway: '172.25.64.1' }
- { name: transient, cidr: '172.25.48.1/20', gateway: '172.25.48.1' }
- { name: openshift, cidr: '172.25.160.1/20', gateway: '172.25.160.1' }
- name: "Connect router's interface to the TENANT-subnet"
neutron_router_interface:
login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin"
@ -1033,6 +1038,7 @@
- { name: qa, prefix: "172.25.112.1/20" }
- { name: scratch, prefix: '172.25.64.1/20' }
- { name: transient, prefix: '172.25.48.1/20' }
- { name: openshift, prefix: '172.25.160.1/20' }
- name: "Create 'web-80-anywhere' security group"
neutron_sec_group:

View file

@ -712,10 +712,8 @@ children:
source_url: https://github.com/collectd/collectd
bugs_url: https://github.com/collectd/collectd/issues
docs_url: https://collectd.org/documentation.shtml
# TODO - write SOP for collectd
# https://fedorahosted.org/fedora-infrastructure/ticket/5161
#sops:
# - https://infrastructure.fedoraproject.org/infra/docs/collectd.rst
- https://infrastructure.fedoraproject.org/infra/docs/collectd.rst
description: >
Tracks and displays statistics on the Fedora
Infrastructure machines over time. Useful for debugging

View file

@ -59,6 +59,20 @@
- autocloud
- autocloud/backend
#
# install koji build fedmsg hotfix
# See issue https://github.com/kushaldas/autocloud/issues/34
#
- name: hotfix - copy over consumer for autocloud
copy: src="{{ files }}/hotfix/autocloud/consumer.py" dest=/usr/lib/python2.7/site-packages/autocloud
owner=root group=root mode=0644
notify:
- restart fedmsg-hub
tags:
- autocloud
- hotfix
- autocloud/backend
- name: install vagrant-libvirt for the libvirt host
dnf: pkg={{ item }} state=present
with_items:

View file

@ -6,6 +6,15 @@
- basset
- basset/frontend
- name: install basset config
template: src=frontend.cfg.j2 dest=/etc/basset/frontend.cfg
owner=basset-frontend group=basset-frontend mode=0600
notify:
- restart httpd
tags:
- basset
- basset/frontend
- name: install staging htpasswd
copy: src={{private}}/files/httpd/basset.stg.htpasswd dest=/etc/httpd/conf.d/basset.htpasswd
owner=root group=root mode=0644

View file

@ -0,0 +1,11 @@
[submission_access]
{% if env == "staging" %}
10.5.126.60 = mediawiki.new,mediawiki.edit
10.5.126.86 = fedora.fas.registration,fedora.fas.cla_sign
{% else %}
10.5.126.63 = mediawiki.new,mediawiki.edit
10.5.126.73 = mediawiki.new,mediawiki.edit
10.5.126.25 = fedora.fas.registration,fedora.fas.cla_sign
10.5.126.26 = fedora.fas.registration,fedora.fas.cla_sign
10.5.126.30 = fedora.fas.registration,fedora.fas.cla_sign
{% endif %}

View file

@ -12,14 +12,12 @@ SECRET_KEY = '{{ mailman_hyperkitty_cookie_key }}'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
TEMPLATE_DEBUG = DEBUG
ADMINS = (
('HyperKitty Admin', 'abompard@fedoraproject.org'),
)
# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
{% for host in mailman_domains %}
"{{ host }}",
@ -47,16 +45,16 @@ MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1')
# Application definition
INSTALLED_APPS = (
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
#'django.contrib.sites',
'django.contrib.messages',
'django.contrib.staticfiles',
# Uncomment the next line to enable the admin:
'django.contrib.admin',
# Uncomment the next line to enable admin documentation:
# 'django.contrib.admindocs',
'hyperkitty',
'social.apps.django_app.default',
'rest_framework',
@ -69,31 +67,59 @@ INSTALLED_APPS = (
'django_extensions',
'postorius',
)
import django
if django.VERSION[:2] < (1, 7):
INSTALLED_APPS = INSTALLED_APPS + ("south",)
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
# Uncomment the next line for simple clickjacking protection:
# 'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.security.SecurityMiddleware',
#'hyperkitty.middleware.SSLRedirect',
'hyperkitty.middleware.TimezoneMiddleware',
'postorius.middleware.PostoriusMiddleware',
)
ROOT_URLCONF = 'urls'
# CSS theme for postorius
MAILMAN_THEME = "default"
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
'{{ mailman_webui_basedir }}/templates',
],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.i18n',
'django.template.context_processors.media',
'django.template.context_processors.static',
'django.template.context_processors.tz',
'django.template.context_processors.csrf',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'social.apps.django_app.context_processors.backends',
'social.apps.django_app.context_processors.login_redirect',
'hyperkitty.context_processors.export_settings',
'hyperkitty.context_processors.postorius_info',
'postorius.context_processors.postorius',
],
},
},
]
WSGI_APPLICATION = 'wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.6/ref/settings/#databases
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
DATABASES = {
'default': {
@ -108,14 +134,14 @@ DATABASES = {
# We're behind a proxy, use the X-Forwarded-Host header
# See https://docs.djangoproject.com/en/1.5/ref/settings/#use-x-forwarded-host
# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
USE_X_FORWARDED_HOST = True
# In the Fedora infra, requests are systematically redirected to HTTPS, so put
# something always true here:
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https')
# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
@ -129,16 +155,7 @@ USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = ''
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = ''
# https://docs.djangoproject.com/en/1.8/howto/static-files/
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
@ -167,54 +184,40 @@ STATICFILES_FINDERS = (
'compressor.finders.CompressorFinder',
)
TEMPLATE_CONTEXT_PROCESSORS = (
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"django.core.context_processors.debug",
"django.core.context_processors.i18n",
"django.core.context_processors.media",
"django.core.context_processors.static",
"django.core.context_processors.csrf",
"django.core.context_processors.request",
"django.core.context_processors.tz",
"django.contrib.messages.context_processors.messages",
"social.apps.django_app.context_processors.backends",
"social.apps.django_app.context_processors.login_redirect",
"hyperkitty.context_processors.export_settings",
"hyperkitty.context_processors.postorius_info",
"postorius.context_processors.postorius",
)
TEMPLATE_DIRS = (
# Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
# Always use forward slashes, even on Windows.
# Don't forget to use absolute paths, not relative paths.
'{{ mailman_webui_basedir }}/templates',
)
# Django 1.6+ defaults to a JSON serializer, but it won't work with django-openid, see
# https://bugs.launchpad.net/django-openid-auth/+bug/1252826
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
LOGIN_URL = 'hk_user_login'
LOGOUT_URL = 'hk_user_logout'
LOGIN_REDIRECT_URL = 'hk_root'
LOGOUT_URL = 'hk_user_logout'
# Use the email as identifier, but truncate it because the User.username field
# is only 30 chars long.
BROWSERID_USERNAME_ALGO = lambda email: email[:30]
# Use the email username as identifier, but truncate it because
# the User.username field is only 30 chars long.
def username(email):
return email.rsplit('@', 1)[0][:30]
BROWSERID_USERNAME_ALGO = username
BROWSERID_VERIFY_CLASS = "django_browserid.views.Verify"
DEFAULT_FROM_EMAIL = "admin@fedoraproject.org"
# Compatibility with Bootstrap 3
from django.contrib.messages import constants as messages
MESSAGE_TAGS = {
messages.ERROR: 'danger'
}
# Django Crispy Forms
CRISPY_TEMPLATE_PACK = 'bootstrap3'
CRISPY_FAIL_SILENTLY = not DEBUG
#
# Social auth
#
AUTHENTICATION_BACKENDS = (
#'social.backends.open_id.OpenIdAuth',
# http://python-social-auth.readthedocs.org/en/latest/backends/google.html
@ -227,12 +230,6 @@ AUTHENTICATION_BACKENDS = (
'django.contrib.auth.backends.ModelBackend',
)
# http://python-social-auth.readthedocs.org/en/latest/configuration/django.html#database
if django.VERSION[:2] < (1, 7):
SOUTH_MIGRATION_MODULES = {
'default': 'social.apps.django_app.default.south_migrations'
}
# http://python-social-auth.readthedocs.org/en/latest/pipeline.html#authentication-pipeline
SOCIAL_AUTH_PIPELINE = (
'social.pipeline.social_auth.social_details',
@ -248,6 +245,7 @@ SOCIAL_AUTH_PIPELINE = (
'social.pipeline.social_auth.associate_user',
'social.pipeline.social_auth.load_extra_data',
'social.pipeline.user.user_details',
'hyperkitty.lib.mailman.add_user_to_mailman',
)
@ -282,16 +280,6 @@ COMPRESS_OFFLINE = True
# needed for debug mode
#INTERNAL_IPS = ('127.0.0.1',)
# Django Crispy Forms
CRISPY_TEMPLATE_PACK = 'bootstrap3'
CRISPY_FAIL_SILENTLY = not DEBUG
# Compatibility with Bootstrap 3
from django.contrib.messages import constants as messages
MESSAGE_TAGS = {
messages.ERROR: 'danger'
}
#
# Full-text search engine
@ -324,7 +312,7 @@ LOGGING = {
'class': 'django.utils.log.AdminEmailHandler'
},
'file':{
'level': 'INFO',
'level': 'DEBUG',
#'class': 'logging.handlers.RotatingFileHandler',
'class': 'logging.handlers.WatchedFileHandler',
'filename': '/var/log/hyperkitty/hyperkitty.log',
@ -333,29 +321,14 @@ LOGGING = {
},
'loggers': {
'django.request': {
'handlers': ['mail_admins'],
'level': 'ERROR',
'propagate': True,
},
'django.request': {
'handlers': ['file'],
'level': 'ERROR',
'propagate': True,
},
'django': {
'handlers': ['file'],
'level': 'ERROR',
'propagate': True,
},
'hyperkitty': {
'handlers': ['file'],
'handlers': ['mail_admins', 'file'],
'level': 'DEBUG',
'propagate': True,
},
},
'formatters': {
'verbose': {
'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
},
'simple': {
'format': '%(levelname)s %(message)s'

View file

@ -36,6 +36,7 @@
- mediawiki119-intersection
- mediawiki119-RSS
- mediawiki-FedoraBadges
- mediawiki119-basset
- php-zmq
- php-pecl-uuid
tags:
@ -43,7 +44,9 @@
- mediawiki
- name: adding FAS auth
template: src=Auth_FAS_CLAPLUSONE.php.j2 dest=/usr/share/mediawiki119/extensions/Auth_FAS.php owner=root group=root mode=775
#template: src=Auth_FAS_CLAPLUSONE.php.j2
template: src=Auth_FAS.php.j2
dest=/usr/share/mediawiki119/extensions/Auth_FAS.php owner=root group=root mode=775
tags:
- config
- mediawiki

View file

@ -324,6 +324,18 @@ require_once "$IP/extensions/fedmsg-emit.php";
require_once "$IP/extensions/HTTP302Found/HTTP302Found.php";
require_once "$IP/extensions/intersection/DynamicPageList.php";
require_once "$IP/extensions/RSS/RSS.php";
require_once "$IP/extensions/BassetSubmitter.php";
{% if env == "staging" %}
$basset_url = 'http://basset01.stg.phx2.fedoraproject.org/basset';
$basset_username = '{{ basset_stg_frontend_user }}';
$basset_password = '{{ basset_stg_frontend_pass }}';
{% else %}
$basset_url = 'http://basset01.phx2.fedoraproject.org/basset';
$basset_username = '{{ basset_prod_frontend_user }}';
$basset_password = '{{ basset_prod_frontend_pass }}';
{% endif %}
$wgShowExceptionDetails = true;