Big Mailman update

inventory/group_vars/mailman

@@ -35,6 +35,25 @@ mailman_db_server: db01.phx2.fedoraproject.org
 mailman_domains:
 - lists.fedoraproject.org
 - lists.fedorahosted.org
+mailman_social_login:
+  gitlab:
+    display_name: GitLab
+    provider: gitlab
+  github:
+    display_name: GitHub
+    provider: github
+  twitter:
+    display_name: Twitter
+    provider: twitter
+  google:
+    display_name: Google
+    provider: google
+  facebook:
+    display_name: Facebook
+    provider: facebook
+  stackexchange:
+    display_name: StackExchange
+    provider: stackexchange
 
 # by default, the number of emails in queue before we whine
 nrpe_check_postfix_queue_warn: 100

inventory/group_vars/mailman-stg

@@ -34,6 +34,25 @@ mailman_db_server: db01.stg.phx2.fedoraproject.org
 mailman_domains:
 - lists.stg.fedoraproject.org
 - lists.stg.fedorahosted.org
+mailman_social_login:
+  gitlab:
+    display_name: GitLab
+    provider: gitlab
+  github:
+    display_name: GitHub
+    provider: github
+  twitter:
+    display_name: Twitter
+    provider: twitter
+  google:
+    display_name: Google
+    provider: google
+  facebook:
+    display_name: Facebook
+    provider: facebook
+  stackexchange:
+    display_name: StackExchange
+    provider: stackexchange
 
 # by default, the number of emails in queue before we whine
 nrpe_check_postfix_queue_warn: 20

roles/mailman/defaults/main.yml

@@ -9,3 +9,4 @@ mailman_hyperkitty_cookie_key: changeme
 mailman_domains:
 - lists.example.com
 - lists.example.org
+mailman_social_login: []

roles/mailman/files/postorius.initial-user.json

@@ -1,20 +0,0 @@
-[
-    {
-        "fields": {
-            "date_joined": "2013-04-10T10:25:49.996", 
-            "email": "root@fedoraproject.org", 
-            "first_name": "List Admin", 
-            "groups": [], 
-            "is_active": true, 
-            "is_staff": true, 
-            "is_superuser": true, 
-            "last_login": "2013-04-10T10:25:49.996", 
-            "last_name": "", 
-            "password": "pbkdf2_sha256$10000$uXnbYx0visOx$yux8mAZ2H/sjyCEYBvO3JEtJ7dmWfNMnvekQjePbue0=", 
-            "user_permissions": [], 
-            "username": "listadmin"
-        }, 
-        "model": "auth.user", 
-        "pk": 1
-    }
-]

roles/mailman/files/urls.py

@@ -1,27 +1,18 @@
 # -*- coding: utf-8 -*-
 
-import hyperkitty
+from django.conf.urls import include, url
-import postorius
-
-from django.conf.urls import patterns, include, url
-from django.conf import settings
-
-# Uncomment the next two lines to enable the admin:
 from django.contrib import admin
-admin.autodiscover()
 
 from django.core.urlresolvers import reverse_lazy
 from django.views.generic import RedirectView
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'^$', RedirectView.as_view(
         url=reverse_lazy('hyperkitty.views.index.index'),
         permanent=True)),
-    #url(r'^$', 'postorius.views.list_index'),
     url(r'^admin/', include('postorius.urls')),
     url(r'^archives/', include('hyperkitty.urls')),
-    # Social Auth
+    url(r'', include('django_mailman3.urls')),
-    url(r'', include('social.apps.django_app.urls', namespace='social')),
+    url(r'^accounts/', include('allauth.urls')),
-    # BrowserID
+    url(r'^django-admin/', include(admin.site.urls)),
-    url(r'', include('django_browserid.urls')),
+]
-)

roles/mailman/tasks/main.yml

@@ -110,8 +110,6 @@
   - python-whoosh
   - python-tox
   - python-vcrpy
-  # django-compressor implicitely requires BeautifulSoup until v1.4
-  - python-BeautifulSoup
   # scripts
   - python34-PyYAML
   # mailman soft dep to convert html to plaintext
@@ -334,7 +332,7 @@
 #
 - name: copy the mailman-hyperkitty conffile
   copy: src=mailman-hyperkitty.cfg
-        dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
+        dest="/etc/mailman3.d/hyperkitty.cfg"
         owner=root group=mailman mode=0640
   tags:
   - config
@@ -342,16 +340,6 @@
   notify:
     - restart mailman3
 
-# To avoid confusion with "{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
-# Don't create a symlink because ansible won't convert the regular file coming
-# from the RPM to a symlink.
-- name: remove the hyperkitty conffile in the mailman directory to avoid confusion
-  file: path=/etc/mailman3.d/hyperkitty.cfg state=absent
-  tags:
-  - config
-  - mailman
-
-
 #
 # Scripts
 #

roles/mailman/templates/initial-data.json.j2

@@ -0,0 +1,30 @@
+[
+{% for host in mailman_domains %}
+{
+    "fields": {
+        "domain": "{{ host }}",
+        "name": "Fedora mailing-lists"
+    },
+    "model": "sites.site",
+    "pk": {{ loop.index }}
+},
+{% endfor %}
+{% for service_name, service_data in mailman_social_login.items() %}
+{
+    "fields": {
+        "name": "{{ service_data.display_name }}",
+        "sites": [
+            {% for host in mailman_domains %}
+            {{ loop.index }}{% if not loop.last %},{% endif %}
+            {% endfor %}
+        ],
+        "client_id": "{{ mailman_social_login_secrets[service_name].client_id }}",
+        "secret": "{{ mailman_social_login_secrets[service_name].secret }}",
+        "key": "",
+        "provider": "{{ service_data.provider }}"
+    },
+    "model": "socialaccount.socialapp",
+    "pk": 1
+}{% if not loop.last %},{% endif %}
+{% endfor %}
+]

roles/mailman/templates/mailman.cfg.j2

@@ -35,7 +35,7 @@ url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_serve
 [archiver.hyperkitty]
 class: mailman_hyperkitty.Archiver
 enable: yes
-configuration: {{ mailman_webui_confdir }}/mailman-hyperkitty.cfg
+configuration: /etc/mailman3.d/hyperkitty.cfg
 
 [archiver.fedmsg]
 class: mailman3_fedmsg_plugin.Archiver

roles/mailman/templates/mailman.cfg.lists-dev.cloud.fedoraproject.org.j2

@@ -32,7 +32,7 @@ pid_file: /run/mailman3/master.pid
 [archiver.hyperkitty]
 class: mailman_hyperkitty.Archiver
 enable: yes
-configuration: {{ mailman_webui_confdir }}/mailman-hyperkitty.cfg
+configuration: /etc/mailman3.d/hyperkitty.cfg
 
 #[archiver.fedmsg]
 #class: mailman3_fedmsg_plugin.Archiver

roles/mailman/templates/post-transaction.action.j2

@@ -4,3 +4,5 @@ hyperkitty:install:{{ mailman_webui_basedir }}/bin/post-update.sh
 hyperkitty:update:{{ mailman_webui_basedir }}/bin/post-update.sh
 postorius:install:{{ mailman_webui_basedir }}/bin/post-update.sh
 postorius:update:{{ mailman_webui_basedir }}/bin/post-update.sh
+python-django-mailman3:install:{{ mailman_webui_basedir }}/bin/post-update.sh
+python-django-mailman3:update:{{ mailman_webui_basedir }}/bin/post-update.sh

roles/mailman/templates/settings.py.j2

@@ -17,6 +17,8 @@ ADMINS = (
 )
 SERVER_EMAIL = 'root@fedoraproject.org'
 
+SITE_ID = 1
+
 # Hosts/domain names that are valid for this site; required if DEBUG is False
 # See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = [
@@ -24,18 +26,10 @@ ALLOWED_HOSTS = [
     "{{ host }}",
 {% endfor %}
     ".fedoraproject.org",
-    "discuss.arquillian.org",
     "localhost", # Archiving API from Mailman
     "127.0.0.1", # HAProxy ping
     "mailman01", # Varnish ping on STG
 ]
-# And for BrowserID too, see
-# http://django-browserid.rtfd.org/page/user/settings.html#django.conf.settings.BROWSERID_AUDIENCES
-BROWSERID_AUDIENCES = [
-{% for host in mailman_domains %}
-    "https://{{ host }}",
-{% endfor %}
-]
 
 # Mailman API credentials
 MAILMAN_REST_API_URL = 'http://localhost:8001'
@@ -54,20 +48,26 @@ INSTALLED_APPS = (
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
-    #'django.contrib.sites',
+    'django.contrib.sites',
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'hyperkitty',
-    'social.apps.django_app.default',
     'rest_framework',
     'django_gravatar',
-    'crispy_forms',
     'paintstore',
     'compressor',
-    'django_browserid',
     'haystack',
     'django_extensions',
     'postorius',
+    'django_mailman3',
+    'allauth',
+    'allauth.account',
+    'allauth.socialaccount',
+    'django_mailman3.lib.auth.fedora',
+    {% for service_name, service_data in mailman_social_login.items() %}
+    'allauth.socialaccount.providers.{{ service_data.provider }}',
+    {% endfor %}
+    'allauth.socialaccount.providers.openid',
 )
 
 
@@ -82,7 +82,7 @@ MIDDLEWARE_CLASSES = (
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'django.middleware.security.SecurityMiddleware',
     #'hyperkitty.middleware.SSLRedirect',
-    'hyperkitty.middleware.TimezoneMiddleware',
+    'django_mailman3.middleware.TimezoneMiddleware',
     'postorius.middleware.PostoriusMiddleware',
 )
 
@@ -95,8 +95,14 @@ TEMPLATES = [
         'DIRS': [
             '{{ mailman_webui_basedir }}/templates',
         ],
-        'APP_DIRS': True,
         'OPTIONS': {
+            'loaders': [
+                # https://docs.djangoproject.com/en/1.8/ref/templates/api/#django.template.loaders.cached.Loader
+                ('django.template.loaders.cached.Loader', [
+                    'django.template.loaders.filesystem.Loader',
+                    'django.template.loaders.app_directories.Loader',
+                ]),
+            ],
             'context_processors': [
                 'django.template.context_processors.debug',
                 'django.template.context_processors.i18n',
@@ -107,8 +113,7 @@ TEMPLATES = [
                 'django.template.context_processors.request',
                 'django.contrib.auth.context_processors.auth',
                 'django.contrib.messages.context_processors.messages',
-                'social.apps.django_app.context_processors.backends',
+                'django_mailman3.context_processors.common',
-                'social.apps.django_app.context_processors.login_redirect',
                 'hyperkitty.context_processors.common',
                 'postorius.context_processors.postorius',
             ],
@@ -185,21 +190,28 @@ STATICFILES_FINDERS = (
     'compressor.finders.CompressorFinder',
 )
 
+
+# Sessions
+# https://docs.djangoproject.com/en/1.8/topics/http/sessions/
+
+SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'
+
+# Security
+SESSION_COOKIE_SECURE = True
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_BROWSER_XSS_FILTER = True
+CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_HTTPONLY = True
+X_FRAME_OPTIONS = 'DENY'
+
 # Django 1.6+ defaults to a JSON serializer, but it won't work with django-openid, see
 # https://bugs.launchpad.net/django-openid-auth/+bug/1252826
-SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+#SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
 
 
-LOGIN_URL          = 'hk_user_login'
+LOGIN_URL          = 'account_login'
 LOGIN_REDIRECT_URL = 'hk_root'
-LOGOUT_URL         = 'hk_user_logout'
+LOGOUT_URL         = 'account_logout'
-
-# Use the email username as identifier, but truncate it because
-# the User.username field is only 30 chars long.
-def username(email):
-    return email.rsplit('@', 1)[0][:30]
-BROWSERID_USERNAME_ALGO = username
-BROWSERID_VERIFY_CLASS = "django_browserid.views.Verify"
 
 
 DEFAULT_FROM_EMAIL = "admin@fedoraproject.org"
@@ -211,44 +223,52 @@ MESSAGE_TAGS = {
     messages.ERROR: 'danger'
     }
 
-# Django Crispy Forms
-CRISPY_TEMPLATE_PACK = 'bootstrap3'
-CRISPY_FAIL_SILENTLY = not DEBUG
-
 
 #
 # Social auth
 #
 AUTHENTICATION_BACKENDS = (
-    #'social.backends.open_id.OpenIdAuth',
-    # http://python-social-auth.readthedocs.org/en/latest/backends/google.html
-    #'social.backends.google.GoogleOpenId',
-    #'social.backends.google.GoogleOAuth2',
-    #'social.backends.twitter.TwitterOAuth',
-    'social.backends.yahoo.YahooOpenId',
-    'social.backends.fedora.FedoraOpenId',
-    'django_browserid.auth.BrowserIDBackend',
     'django.contrib.auth.backends.ModelBackend',
+    'allauth.account.auth_backends.AuthenticationBackend',
 )
 
-# http://python-social-auth.readthedocs.org/en/latest/pipeline.html#authentication-pipeline
-SOCIAL_AUTH_PIPELINE = (
-    'social.pipeline.social_auth.social_details',
-    'social.pipeline.social_auth.social_uid',
-    'social.pipeline.social_auth.auth_allowed',
-    'social.pipeline.social_auth.social_user',
-    'social.pipeline.user.get_username',
-    # Associates the current social details with another user account with
-    # a similar email address. Disabled by default, enable with care:
-    # http://python-social-auth.readthedocs.org/en/latest/use_cases.html#associate-users-by-email
-    'social.pipeline.social_auth.associate_by_email',
-    'social.pipeline.user.create_user',
-    'social.pipeline.social_auth.associate_user',
-    'social.pipeline.social_auth.load_extra_data',
-    'social.pipeline.user.user_details',
-    'hyperkitty.lib.mailman.add_user_to_mailman',
-)
 
+# Django Allauth
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+ACCOUNT_UNIQUE_EMAIL  = True
+
+SOCIALACCOUNT_PROVIDERS = {
+    'openid': {
+        'SERVERS': [
+            dict(id='yahoo',
+                 name='Yahoo',
+                 openid_url='http://me.yahoo.com'),
+        ],
+    },
+    'google': {
+        'SCOPE': ['profile', 'email'],
+        'AUTH_PARAMS': {'access_type': 'online'},
+    },
+    'facebook': {
+       'METHOD': 'oauth2',
+       'SCOPE': ['email'],
+       'FIELDS': [
+           'email',
+           'name',
+           'first_name',
+           'last_name',
+           'locale',
+           'timezone',
+           ],
+       'VERSION': 'v2.4',
+    },
+    'stackexchange': {
+        'SITE': 'stackoverflow',
+    },
+}
 
 
 #
@@ -351,26 +371,9 @@ CACHES = {
 }
 
 
-#
-# HyperKitty-specific
-#
-
-APP_NAME = 'Fedora list archives'
-
-# Allow authentication with the internal user database?
-# By default, only a login through Persona or your email provider is allowed.
-USE_INTERNAL_AUTH = False
-
-# Use SSL when logged in. You need to enable the SSLRedirect middleware for
-# this feature to work.
-#USE_SSL = True
-
 # Only display mailing-lists from the same virtual host as the webserver
 FILTER_VHOST = False
 
-# This is for development purposes
-USE_MOCKUPS = False
-
 
 try:
     from settings_local import *