From b3c934f34943aa9b329de76181b381ded8b2ff7b Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Tue, 22 Mar 2016 21:04:20 +0000 Subject: [PATCH 01/14] Add collectd SOP to apps --- roles/apps-fp-o/files/apps.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/apps-fp-o/files/apps.yaml b/roles/apps-fp-o/files/apps.yaml index 6b5d599e2e..24309c661e 100644 --- a/roles/apps-fp-o/files/apps.yaml +++ b/roles/apps-fp-o/files/apps.yaml @@ -712,10 +712,8 @@ children: source_url: https://github.com/collectd/collectd bugs_url: https://github.com/collectd/collectd/issues docs_url: https://collectd.org/documentation.shtml - # TODO - write SOP for collectd - # https://fedorahosted.org/fedora-infrastructure/ticket/5161 #sops: - # - https://infrastructure.fedoraproject.org/infra/docs/collectd.rst + - https://infrastructure.fedoraproject.org/infra/docs/collectd.rst description: > Tracks and displays statistics on the Fedora Infrastructure machines over time. Useful for debugging From 1cacb25fd40f7026cda0ad0fdbb1054c7a610ca3 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 20:44:02 +0000 Subject: [PATCH 02/14] Enable mediawiki basset plugin Signed-off-by: Patrick Uiterwijk --- roles/mediawiki/tasks/main.yml | 1 + roles/mediawiki/templates/LocalSettings.php.fp.j2 | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index dc646837a5..ddf1e8e558 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -36,6 +36,7 @@ - mediawiki119-intersection - mediawiki119-RSS - mediawiki-FedoraBadges + - mediawiki119-basset - php-zmq - php-pecl-uuid tags: diff --git a/roles/mediawiki/templates/LocalSettings.php.fp.j2 b/roles/mediawiki/templates/LocalSettings.php.fp.j2 index a6c10b43f4..3362115363 100644 --- a/roles/mediawiki/templates/LocalSettings.php.fp.j2 +++ b/roles/mediawiki/templates/LocalSettings.php.fp.j2 @@ -324,6 +324,18 @@ require_once "$IP/extensions/fedmsg-emit.php"; require_once "$IP/extensions/HTTP302Found/HTTP302Found.php"; require_once "$IP/extensions/intersection/DynamicPageList.php"; require_once "$IP/extensions/RSS/RSS.php"; +require_once "$IP/extensions/BassetSubmitter.php"; + +{% if env == "staging" %} +$basset_url = 'http://basset01.stg.phx2.fedoraproject.org/basset'; +$basset_username = '{{ basset_stg_frontend_user }}'; +$basset_password = '{{ basset_stg_frontend_pass }}'; +{% else %} +$basset_url = 'http://basset01.phx2.fedoraproject.org/basset'; +$basset_username = '{{ basset_prod_frontend_user }}'; +$basset_password = '{{ basset_prod_frontend_pass }}'; +{% endif %} + $wgShowExceptionDetails = true; From 1d79f5e85a50c523e129f82b8e4edffedba42b44 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 20:46:39 +0000 Subject: [PATCH 03/14] Open iptables on basset for wiki Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/basset | 3 +++ inventory/group_vars/basset-stg | 2 ++ 2 files changed, 5 insertions(+) diff --git a/inventory/group_vars/basset b/inventory/group_vars/basset index 5bab0fa8b4..85fc0692a8 100644 --- a/inventory/group_vars/basset +++ b/inventory/group_vars/basset @@ -9,6 +9,9 @@ custom_rules: [ '-A INPUT -p tcp -m tcp -s 10.5.126.25 --dport 80 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 10.5.126.26 --dport 80 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 10.5.126.30 --dport 80 -j ACCEPT' + # wiki01, wiki02 + '-A INPUT -p tcp -m tcp -s 10.5.126.63 --dport 80 -j ACCEPT' + '-A INPUT -p tcp -m tcp -s 10.5.126.73 --dport 80 -j ACCEPT' ] fas_client_groups: sysadmin-main diff --git a/inventory/group_vars/basset-stg b/inventory/group_vars/basset-stg index ad45a6f96f..02049e5279 100644 --- a/inventory/group_vars/basset-stg +++ b/inventory/group_vars/basset-stg @@ -7,6 +7,8 @@ num_cpus: 2 custom_rules: [ # fas01.stg '-A INPUT -p tcp -m tcp -s 10.5.126.86 --dport 80 -j ACCEPT', + # wiki01.stg + '-A INPUT -p tcp -m tcp -s 10.5.126.60 --dport 80 -j ACCEPT', ] fas_client_groups: sysadmin-main From e7593a12f1731c8badad6315da101d256afa1d5b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 20:51:07 +0000 Subject: [PATCH 04/14] Add ACL for basset frontend Signed-off-by: Patrick Uiterwijk --- roles/basset/frontend/tasks/main.yml | 9 +++++++++ roles/basset/frontend/templates/frontend.cfg.j2 | 6 ++++++ 2 files changed, 15 insertions(+) create mode 100644 roles/basset/frontend/templates/frontend.cfg.j2 diff --git a/roles/basset/frontend/tasks/main.yml b/roles/basset/frontend/tasks/main.yml index 54637ab228..d2ccf6b303 100644 --- a/roles/basset/frontend/tasks/main.yml +++ b/roles/basset/frontend/tasks/main.yml @@ -6,6 +6,15 @@ - basset - basset/frontend +- name: install basset config + template: src=frontend.cfg.j2 dest=/etc/basset/frontend.cfg + user=basset-frontend group=basset-frontend mode=0600 + notify: + - restart httpd + tags: + - basset + - basset/frontend + - name: install staging htpasswd copy: src={{private}}/files/httpd/basset.stg.htpasswd dest=/etc/httpd/conf.d/basset.htpasswd owner=root group=root mode=0644 diff --git a/roles/basset/frontend/templates/frontend.cfg.j2 b/roles/basset/frontend/templates/frontend.cfg.j2 new file mode 100644 index 0000000000..476efa2737 --- /dev/null +++ b/roles/basset/frontend/templates/frontend.cfg.j2 @@ -0,0 +1,6 @@ +[submission_access] +10.5.126.63 = mediawiki.new,mediawiki.edit +10.5.126.73 = mediawiki.new,mediawiki.edit +10.5.126.25 = fedora.fas.registration,fedora.fas.cla_sign +10.5.126.26 = fedora.fas.registration,fedora.fas.cla_sign +10.5.126.30 = fedora.fas.registration,fedora.fas.cla_sign From bac647a47ea34501e976eac531f116aa5807b4b5 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 21:25:25 +0000 Subject: [PATCH 05/14] Commas are required Signed-off-by: Patrick Uiterwijk --- inventory/group_vars/basset | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inventory/group_vars/basset b/inventory/group_vars/basset index 85fc0692a8..8b21f69607 100644 --- a/inventory/group_vars/basset +++ b/inventory/group_vars/basset @@ -8,10 +8,10 @@ custom_rules: [ # fas01, fas02, and fas03 '-A INPUT -p tcp -m tcp -s 10.5.126.25 --dport 80 -j ACCEPT', '-A INPUT -p tcp -m tcp -s 10.5.126.26 --dport 80 -j ACCEPT', - '-A INPUT -p tcp -m tcp -s 10.5.126.30 --dport 80 -j ACCEPT' + '-A INPUT -p tcp -m tcp -s 10.5.126.30 --dport 80 -j ACCEPT', # wiki01, wiki02 - '-A INPUT -p tcp -m tcp -s 10.5.126.63 --dport 80 -j ACCEPT' - '-A INPUT -p tcp -m tcp -s 10.5.126.73 --dport 80 -j ACCEPT' + '-A INPUT -p tcp -m tcp -s 10.5.126.63 --dport 80 -j ACCEPT', + '-A INPUT -p tcp -m tcp -s 10.5.126.73 --dport 80 -j ACCEPT', ] fas_client_groups: sysadmin-main From 4b893b9f2bc845e20ed11fa03a30f4f440a99b2e Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 21:32:37 +0000 Subject: [PATCH 06/14] Also add authz for staging Signed-off-by: Patrick Uiterwijk --- roles/basset/frontend/templates/frontend.cfg.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/basset/frontend/templates/frontend.cfg.j2 b/roles/basset/frontend/templates/frontend.cfg.j2 index 476efa2737..47fbf97d76 100644 --- a/roles/basset/frontend/templates/frontend.cfg.j2 +++ b/roles/basset/frontend/templates/frontend.cfg.j2 @@ -1,6 +1,11 @@ [submission_access] +{% if env == "staging" %} +10.5.126.60 = mediawiki.new,mediawiki.edit +10.5.126.86 = fedora.fas.registration,fedora.fas.cla_sign +{% else %} 10.5.126.63 = mediawiki.new,mediawiki.edit 10.5.126.73 = mediawiki.new,mediawiki.edit 10.5.126.25 = fedora.fas.registration,fedora.fas.cla_sign 10.5.126.26 = fedora.fas.registration,fedora.fas.cla_sign 10.5.126.30 = fedora.fas.registration,fedora.fas.cla_sign +{% endif %} From c259217e16b746711a19f22943225b779dbae1c7 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 21:34:51 +0000 Subject: [PATCH 07/14] You would think I'd know by now that this is owner Signed-off-by: Patrick Uiterwijk --- roles/basset/frontend/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/basset/frontend/tasks/main.yml b/roles/basset/frontend/tasks/main.yml index d2ccf6b303..6ae7f046b3 100644 --- a/roles/basset/frontend/tasks/main.yml +++ b/roles/basset/frontend/tasks/main.yml @@ -8,7 +8,7 @@ - name: install basset config template: src=frontend.cfg.j2 dest=/etc/basset/frontend.cfg - user=basset-frontend group=basset-frontend mode=0600 + owner=basset-frontend group=basset-frontend mode=0600 notify: - restart httpd tags: From 984049f520b0355149cd2c74c1bb322956b0d55d Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Tue, 22 Mar 2016 22:28:40 +0000 Subject: [PATCH 08/14] Remove +1 requirement from wiki, leaving CLA Signed-off-by: Patrick Uiterwijk --- roles/mediawiki/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/mediawiki/tasks/main.yml b/roles/mediawiki/tasks/main.yml index ddf1e8e558..ee237598d9 100644 --- a/roles/mediawiki/tasks/main.yml +++ b/roles/mediawiki/tasks/main.yml @@ -44,7 +44,9 @@ - mediawiki - name: adding FAS auth - template: src=Auth_FAS_CLAPLUSONE.php.j2 dest=/usr/share/mediawiki119/extensions/Auth_FAS.php owner=root group=root mode=775 + #template: src=Auth_FAS_CLAPLUSONE.php.j2 + template: src=Auth_FAS.php.j2 + dest=/usr/share/mediawiki119/extensions/Auth_FAS.php owner=root group=root mode=775 tags: - config - mediawiki From f7b1bf59d02e13a35868c04c3f0a3098dfaf4c68 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 23 Mar 2016 11:15:55 +0000 Subject: [PATCH 09/14] Add tenant for openshift and account for misc Signed-off-by: Patrick Uiterwijk --- playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml index f350205226..4248e09b86 100644 --- a/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml +++ b/playbooks/hosts/fed-cloud09.cloud.fedoraproject.org.yml @@ -24,7 +24,7 @@ vars: # this is actually without admin tenant all_tenants: ['cloudintern', 'cloudsig', 'copr', 'coprdev', 'infrastructure', - 'persistent', 'pythonbots', 'qa', 'scratch', 'transient'] + 'persistent', 'pythonbots', 'qa', 'scratch', 'transient', 'openshift'] vars_files: - /srv/web/infra/ansible/vars/global.yml @@ -688,6 +688,7 @@ - { name: coprdev, desc: 'Development version of Copr' } - { name: pythonbots, desc: 'project for python build bot users - twisted, etc' } - { name: scratch, desc: 'scratch and short term instances' } + - { name: openshift, desc: 'Tenant for openshift deployment' } ##### USERS ##### @@ -727,6 +728,7 @@ - { name: roshi, email: 'roshi@fedoraproject.org', tenant: qa, password: "{{roshi_password}}" } - { name: maxamillion, email: 'maxamillion@fedoraproject.org', tenant: infrastructure, password: "{{maxamillion_password}}" } - { name: clime, email: 'clime@redhat.com', tenant: copr, password: "{{clime_password}}" } + - { name: misc, email: 'misc@redhat.com', tenant: openshift, password: "{{misc_password}}" } tags: - openstack_users @@ -765,6 +767,7 @@ - { username: admin, name: fedora-admin-20130801, tenant: admin, password: "{{ADMIN_PASS}}", public_key: "{{ lookup('file', files + '/fedora-cloud/fedora-admin-20130801.pub') }}" } - { username: asamalik, name: asamalik, tenant: scratch, password: "{{asamalik_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas asamalik') }}" } - { username: clime, name: clime, tenant: copr, password: "{{clime_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas clime') }}" } + - { username: misc, name: misc, tenant: openshift, password: "{{misc_password}}", public_key: "{{ lookup('pipe', '/srv/web/infra/ansible/scripts/auth-keys-from-fas misc') }}" } tags: - openstack_users @@ -907,6 +910,7 @@ - { name: qa, shared: false } - { name: scratch, shared: false } - { name: transient, shared: false } + - { name: openshift, shared: false } - name: Create a subnet for all tenants neutron_subnet: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" @@ -928,6 +932,7 @@ - { name: qa, cidr: '172.25.112.1/20', gateway: '172.25.112.1' } - { name: scratch, cidr: '172.25.64.1/20', gateway: '172.25.64.1' } - { name: transient, cidr: '172.25.48.1/20', gateway: '172.25.48.1' } + - { name: openshift, cidr: '172.25.160.1/20', gateway: '172.25.160.1' } - name: "Connect router's interface to the TENANT-subnet" neutron_router_interface: login_username="admin" login_password="{{ ADMIN_PASS }}" login_tenant_name="admin" @@ -1033,6 +1038,7 @@ - { name: qa, prefix: "172.25.112.1/20" } - { name: scratch, prefix: '172.25.64.1/20' } - { name: transient, prefix: '172.25.48.1/20' } + - { name: openshift, prefix: '172.25.160.1/20' } - name: "Create 'web-80-anywhere' security group" neutron_sec_group: From 94f903499b48d3e52bd26d261165af5679342ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Wed, 23 Mar 2016 14:59:01 +0000 Subject: [PATCH 10/14] HyperKitty: update the Django settings file --- roles/mailman/templates/settings.py.j2 | 159 ++++++++++--------------- 1 file changed, 66 insertions(+), 93 deletions(-) diff --git a/roles/mailman/templates/settings.py.j2 b/roles/mailman/templates/settings.py.j2 index e5e84b1882..12140e669a 100644 --- a/roles/mailman/templates/settings.py.j2 +++ b/roles/mailman/templates/settings.py.j2 @@ -12,14 +12,12 @@ SECRET_KEY = '{{ mailman_hyperkitty_cookie_key }}' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False -TEMPLATE_DEBUG = DEBUG - ADMINS = ( ('HyperKitty Admin', 'abompard@fedoraproject.org'), ) # Hosts/domain names that are valid for this site; required if DEBUG is False -# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts +# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts ALLOWED_HOSTS = [ {% for host in mailman_domains %} "{{ host }}", @@ -47,16 +45,16 @@ MAILMAN_ARCHIVER_FROM = ('127.0.0.1', '::1') # Application definition INSTALLED_APPS = ( + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', #'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', - # Uncomment the next line to enable the admin: - 'django.contrib.admin', - # Uncomment the next line to enable admin documentation: - # 'django.contrib.admindocs', 'hyperkitty', 'social.apps.django_app.default', 'rest_framework', @@ -69,31 +67,59 @@ INSTALLED_APPS = ( 'django_extensions', 'postorius', ) -import django -if django.VERSION[:2] < (1, 7): - INSTALLED_APPS = INSTALLED_APPS + ("south",) MIDDLEWARE_CLASSES = ( - 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - # Uncomment the next line for simple clickjacking protection: - # 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', #'hyperkitty.middleware.SSLRedirect', 'hyperkitty.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', ) ROOT_URLCONF = 'urls' -# CSS theme for postorius -MAILMAN_THEME = "default" + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + '{{ mailman_webui_basedir }}/templates', + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'social.apps.django_app.context_processors.backends', + 'social.apps.django_app.context_processors.login_redirect', + 'hyperkitty.context_processors.export_settings', + 'hyperkitty.context_processors.postorius_info', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' # Database -# https://docs.djangoproject.com/en/1.6/ref/settings/#databases +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases DATABASES = { 'default': { @@ -108,14 +134,14 @@ DATABASES = { # We're behind a proxy, use the X-Forwarded-Host header -# See https://docs.djangoproject.com/en/1.5/ref/settings/#use-x-forwarded-host +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host USE_X_FORWARDED_HOST = True # In the Fedora infra, requests are systematically redirected to HTTPS, so put # something always true here: SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https') # Internationalization -# https://docs.djangoproject.com/en/1.6/topics/i18n/ +# https://docs.djangoproject.com/en/1.8/topics/i18n/ LANGUAGE_CODE = 'en-us' @@ -129,16 +155,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.6/howto/static-files/ - -# Absolute filesystem path to the directory that will hold user-uploaded files. -# Example: "/var/www/example.com/media/" -MEDIA_ROOT = '' - -# URL that handles the media served from MEDIA_ROOT. Make sure to use a -# trailing slash. -# Examples: "http://example.com/media/", "http://media.example.com/" -MEDIA_URL = '' +# https://docs.djangoproject.com/en/1.8/howto/static-files/ # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files @@ -167,54 +184,40 @@ STATICFILES_FINDERS = ( 'compressor.finders.CompressorFinder', ) - -TEMPLATE_CONTEXT_PROCESSORS = ( - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - "django.core.context_processors.debug", - "django.core.context_processors.i18n", - "django.core.context_processors.media", - "django.core.context_processors.static", - "django.core.context_processors.csrf", - "django.core.context_processors.request", - "django.core.context_processors.tz", - "django.contrib.messages.context_processors.messages", - "social.apps.django_app.context_processors.backends", - "social.apps.django_app.context_processors.login_redirect", - "hyperkitty.context_processors.export_settings", - "hyperkitty.context_processors.postorius_info", - "postorius.context_processors.postorius", -) - -TEMPLATE_DIRS = ( - # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". - # Always use forward slashes, even on Windows. - # Don't forget to use absolute paths, not relative paths. - '{{ mailman_webui_basedir }}/templates', -) - # Django 1.6+ defaults to a JSON serializer, but it won't work with django-openid, see # https://bugs.launchpad.net/django-openid-auth/+bug/1252826 SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' LOGIN_URL = 'hk_user_login' -LOGOUT_URL = 'hk_user_logout' LOGIN_REDIRECT_URL = 'hk_root' +LOGOUT_URL = 'hk_user_logout' -# Use the email as identifier, but truncate it because the User.username field -# is only 30 chars long. -BROWSERID_USERNAME_ALGO = lambda email: email[:30] +# Use the email username as identifier, but truncate it because +# the User.username field is only 30 chars long. +def username(email): + return email.rsplit('@', 1)[0][:30] +BROWSERID_USERNAME_ALGO = username BROWSERID_VERIFY_CLASS = "django_browserid.views.Verify" DEFAULT_FROM_EMAIL = "admin@fedoraproject.org" +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages +MESSAGE_TAGS = { + messages.ERROR: 'danger' + } + +# Django Crispy Forms +CRISPY_TEMPLATE_PACK = 'bootstrap3' +CRISPY_FAIL_SILENTLY = not DEBUG + + # # Social auth # - AUTHENTICATION_BACKENDS = ( #'social.backends.open_id.OpenIdAuth', # http://python-social-auth.readthedocs.org/en/latest/backends/google.html @@ -227,12 +230,6 @@ AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', ) -# http://python-social-auth.readthedocs.org/en/latest/configuration/django.html#database -if django.VERSION[:2] < (1, 7): - SOUTH_MIGRATION_MODULES = { - 'default': 'social.apps.django_app.default.south_migrations' - } - # http://python-social-auth.readthedocs.org/en/latest/pipeline.html#authentication-pipeline SOCIAL_AUTH_PIPELINE = ( 'social.pipeline.social_auth.social_details', @@ -248,6 +245,7 @@ SOCIAL_AUTH_PIPELINE = ( 'social.pipeline.social_auth.associate_user', 'social.pipeline.social_auth.load_extra_data', 'social.pipeline.user.user_details', + 'hyperkitty.lib.mailman.add_user_to_mailman', ) @@ -282,16 +280,6 @@ COMPRESS_OFFLINE = True # needed for debug mode #INTERNAL_IPS = ('127.0.0.1',) -# Django Crispy Forms -CRISPY_TEMPLATE_PACK = 'bootstrap3' -CRISPY_FAIL_SILENTLY = not DEBUG - -# Compatibility with Bootstrap 3 -from django.contrib.messages import constants as messages -MESSAGE_TAGS = { - messages.ERROR: 'danger' - } - # # Full-text search engine @@ -324,7 +312,7 @@ LOGGING = { 'class': 'django.utils.log.AdminEmailHandler' }, 'file':{ - 'level': 'INFO', + 'level': 'DEBUG', #'class': 'logging.handlers.RotatingFileHandler', 'class': 'logging.handlers.WatchedFileHandler', 'filename': '/var/log/hyperkitty/hyperkitty.log', @@ -333,29 +321,14 @@ LOGGING = { }, 'loggers': { 'django.request': { - 'handlers': ['mail_admins'], - 'level': 'ERROR', - 'propagate': True, - }, - 'django.request': { - 'handlers': ['file'], - 'level': 'ERROR', - 'propagate': True, - }, - 'django': { - 'handlers': ['file'], - 'level': 'ERROR', - 'propagate': True, - }, - 'hyperkitty': { - 'handlers': ['file'], + 'handlers': ['mail_admins', 'file'], 'level': 'DEBUG', 'propagate': True, }, }, 'formatters': { 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' }, 'simple': { 'format': '%(levelname)s %(message)s' From 30fbcf62238a80153be3d32b31b6d55fc8fa46a8 Mon Sep 17 00:00:00 2001 From: Kevin Fenzi Date: Wed, 23 Mar 2016 17:33:38 +0000 Subject: [PATCH 11/14] Mount prod koji ro on composer.stg --- playbooks/groups/composers.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/groups/composers.yml b/playbooks/groups/composers.yml index 5978d22079..4f198e7aba 100644 --- a/playbooks/groups/composers.yml +++ b/playbooks/groups/composers.yml @@ -35,6 +35,11 @@ mnt_dir: '/mnt/fedora_koji' nfs_src_dir: 'fedora_koji' when: datacenter == 'staging' + - role: nfs/client + mnt_dir: '/mnt/fedora_koji_prod' + nfs_src_dir: 'fedora_koji' + nfs_mount_opts: 'ro,hard,bg,intr,noatime,nodev,nosuid,nfsvers=3' + when: datacenter == 'staging' - releng - fedmsg/base - sudo From 88c961a86e46e67c2fc77ce970634775eac59d86 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Wed, 23 Mar 2016 19:04:42 +0000 Subject: [PATCH 12/14] Original consumer.py for the autocloud hotfix --- files/hotfix/autocloud/consumer.py | 118 +++++++++++++++++++++++++ roles/autocloud/backend/tasks/main.yml | 14 +++ 2 files changed, 132 insertions(+) create mode 100644 files/hotfix/autocloud/consumer.py diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py new file mode 100644 index 0000000000..8f9bf95974 --- /dev/null +++ b/files/hotfix/autocloud/consumer.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +import fedmsg.consumers +import koji + +from autocloud.utils import get_image_url, produce_jobs, get_image_name +import autocloud + +import logging +log = logging.getLogger("fedmsg") + +DEBUG = autocloud.DEBUG + + +class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): + + if DEBUG: + topic = 'org.fedoraproject.dev.__main__.buildsys.task.state.change' + else: + topic = 'org.fedoraproject.prod.buildsys.task.state.change' + + config_key = 'autocloud.consumer.enabled' + + def __init__(self, *args, **kwargs): + super(AutoCloudConsumer, self).__init__(*args, **kwargs) + + def _get_tasks(self, builds): + """ Takes a list of koji createImage task IDs and returns dictionary of + build ids and image url corresponding to that build ids""" + + if autocloud.VIRTUALBOX: + _supported_images = ('Fedora-Cloud-Base-Vagrant', + 'Fedora-Cloud-Atomic-Vagrant',) + else: + _supported_images = ('Fedora-Cloud-Base-Vagrant', + 'Fedora-Cloud-Atomic-Vagrant', + 'Fedora-Cloud-Atomic', 'Fedora-Cloud-Base',) + + for build in builds: + log.info('Got Koji build {0}'.format(build)) + + # Create a Koji connection to the Fedora Koji instance + koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL) + + image_files = [] # list of full URLs of files + + if len(builds) == 1: + task_result = koji_session.getTaskResult(builds[0]) + name = task_result.get('name') + #TODO: Change to get the release information from PDC instead + # of koji once it is set up + release = task_result.get('version') + if name in _supported_images: + task_relpath = koji.pathinfo.taskrelpath(int(builds[0])) + url = get_image_url(task_result.get('files'), task_relpath) + if url: + name = get_image_name(image_name=name) + data = { + 'buildid': builds[0], + 'image_url': url, + 'name': name, + 'release': release, + } + image_files.append(data) + elif len(builds) >= 2: + koji_session.multicall = True + for build in builds: + koji_session.getTaskResult(build) + results = koji_session.multiCall() + for result in results: + + if not result: + continue + + name = result[0].get('name') + if name not in _supported_images: + continue + + #TODO: Change to get the release information from PDC instead + # of koji once it is set up + release = result[0].get('version') + task_relpath = koji.pathinfo.taskrelpath( + int(result[0].get('task_id'))) + url = get_image_url(result[0].get('files'), task_relpath) + if url: + name = get_image_name(image_name=name) + data = { + 'buildid': result[0]['task_id'], + 'image_url': url, + 'name': name, + 'release': release, + } + image_files.append(data) + + return image_files + + def consume(self, msg): + """ This is called when we receive a message matching the topic. """ + + builds = list() # These will be the Koji build IDs to upload, if any. + + msg_info = msg["body"]["msg"]["info"] + + log.info('Received %r %r' % (msg['topic'], msg['body']['msg_id'])) + + # If the build method is "image", we check to see if the child + # task's method is "createImage". + if msg_info["method"] == "image": + if isinstance(msg_info["children"], list): + for child in msg_info["children"]: + if child["method"] == "createImage": + # We only care about the image if the build + # completed successfully (with state code 2). + if child["state"] == 2: + builds.append(child["id"]) + + if len(builds) > 0: + produce_jobs(self._get_tasks(builds)) diff --git a/roles/autocloud/backend/tasks/main.yml b/roles/autocloud/backend/tasks/main.yml index 77bd0ec269..7ea2266a18 100644 --- a/roles/autocloud/backend/tasks/main.yml +++ b/roles/autocloud/backend/tasks/main.yml @@ -59,6 +59,20 @@ - autocloud - autocloud/backend +# +# install koji build fedmsg hotfix +# See issue https://github.com/kushaldas/autocloud/issues/34 +# +- name: hotfix - copy over consumer for autocloud + copy: src="{{ files }}/hotfix/autocloud/consumer.py" dest=/usr/lib/python2.7/site-packages/autocloud + owner=root group=root mode=0644 + notify: + - restart fedmsg-hub + tags: + - autocloud + - hotfix + - autocloud/backend + - name: install vagrant-libvirt for the libvirt host dnf: pkg={{ item }} state=present with_items: From 9992275d3b8aa60ee8592bf9bb4949fbad9f07d3 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Wed, 23 Mar 2016 19:06:36 +0000 Subject: [PATCH 13/14] Hotfix patch for autocloud --- files/hotfix/autocloud/consumer.py | 49 ++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/files/hotfix/autocloud/consumer.py b/files/hotfix/autocloud/consumer.py index 8f9bf95974..c70cde9841 100644 --- a/files/hotfix/autocloud/consumer.py +++ b/files/hotfix/autocloud/consumer.py @@ -15,9 +15,16 @@ DEBUG = autocloud.DEBUG class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): if DEBUG: - topic = 'org.fedoraproject.dev.__main__.buildsys.task.state.change' + topic = [ + 'org.fedoraproject.dev.__main__.buildsys.build.state.change', + 'org.fedoraproject.dev.__main__.buildsys.task.state.change', + ] + else: - topic = 'org.fedoraproject.prod.buildsys.task.state.change' + topic = [ + 'org.fedoraproject.prod.buildsys.build.state.change', + 'org.fedoraproject.prod.buildsys.task.state.change', + ] config_key = 'autocloud.consumer.enabled' @@ -97,6 +104,44 @@ class AutoCloudConsumer(fedmsg.consumers.FedmsgConsumer): def consume(self, msg): """ This is called when we receive a message matching the topic. """ + if msg['topic'].endswith('.buildsys.task.state.change'): + # Do the thing you've always done... this will go away soon. + # releng is transitioning away from it. + self._consume_scratch_task(msg) + elif msg['topic'].endswith('.buildsys.build.state.change'): + # Do the new thing we need to do. handle a 'real build' from koji, + # not just a scratch task. + self._consume_real_build(msg) + else: + raise NotImplementedError("Should be impossible to get here...") + + def _consume_real_build(self, msg): + builds = list() # These will be the Koji task IDs to upload, if any. + + msg = msg['body']['msg'] + if msg['owner'] != 'releng': + log.debug("Dropping message. Owned by %r" % msg['owner']) + return + + if msg['instance'] != 'primary': + log.info("Dropping message. From %r instance." % msg['instance']) + return + + # Don't upload *any* images if one of them fails. + if msg['new'] != 1: + log.info("Dropping message. State is %r" % msg['new']) + return + + koji_session = koji.ClientSession(autocloud.KOJI_SERVER_URL) + children = koji_session.getTaskChildren(msg['task_id']) + for child in children: + if child["method"] == "createImage": + builds.append(child["id"]) + + if len(builds) > 0: + produce_jobs(self._get_tasks(builds)) + + def _consume_scratch_task(self, msg): builds = list() # These will be the Koji build IDs to upload, if any. msg_info = msg["body"]["msg"]["info"] From e15e88022132da8eb9a5203eaf8cda270f1147af Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Wed, 23 Mar 2016 19:14:43 +0000 Subject: [PATCH 14/14] Consider the qa network too when building the stg fedmsg routing policy. --- filter_plugins/fedmsg.py | 2 +- inventory/inventory | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/filter_plugins/fedmsg.py b/filter_plugins/fedmsg.py index de31a2a174..60beaf8bce 100644 --- a/filter_plugins/fedmsg.py +++ b/filter_plugins/fedmsg.py @@ -9,7 +9,7 @@ def invert_fedmsg_policy(groups, vars, env): """ if env == 'staging': - hosts = groups['staging'] + hosts = groups['staging'] + groups['fedmsg-qa-network-stg'] else: hosts = [h for h in groups['all'] if h not in groups['staging']] diff --git a/inventory/inventory b/inventory/inventory index dc4acc0a3b..e3208687d5 100644 --- a/inventory/inventory +++ b/inventory/inventory @@ -797,6 +797,7 @@ zanata2fedmsg01.phx2.fedoraproject.org # See also: # - inventory/group_vars/proxies for the iptables custom_rules list # - roles/fedmsg/base/templates/relay.py.j2 +# - filter_plugins/fedmsg.py [fedmsg-qa-network] retrace01.qa.fedoraproject.org retrace02.qa.fedoraproject.org