Mailman: improve staging-sync script, add manage.py

roles/mailman/files/post-update.sh

@@ -39,4 +39,5 @@ echo "unit tests"
 django-admin test --pythonpath $CONFDIR --settings settings_test django_mailman3 hyperkitty postorius
 
 # Restart services
+echo "Start services"
 systemctl start httpd mailman3 crond webui-qcluster

roles/mailman/files/prod-to-stg.py

@@ -11,7 +11,7 @@ from subprocess import call
 from urlparse import urlparse
 
 if not os.getenv("DJANGO_SETTINGS_MODULE"):
-    os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
+    os.environ["DJANGO_SETTINGS_MODULE"] = "settings_admin"
 sys.path.insert(0, "/srv/webui/config")
 
 import django
@@ -22,9 +22,9 @@ MAILMAN_TABLES_TO_REPLACE = [
     ("domain", "mail_host"),
     ("mailinglist", "mail_host"),
     ("mailinglist", "list_id"),
-    ("ban", "list_id"),
+    #("ban", "list_id"),
-    ("bounceevent", "list_id"),
+    #("bounceevent", "list_id"),
-    ("member", "list_id"),
+    #("member", "list_id"),
     ("template", "context"),
     ("acceptablealias", "alias"),
 ]
@@ -37,17 +37,56 @@ DJANGO_TABLES_TO_EMPTY = [
     "social_auth_usersocialauth",
     "socialaccount_socialtoken",
 ]
+DJANGO_INDICES_TO_RECREATE = [
+    ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id", "(mailinglist_id)"),
+    ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
+    ("hyperkitty_email", "hyperkitty_email_mailinglist_id", "(mailinglist_id)"),
+    ("hyperkitty_email", "hyperkitty_email_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
+]
+DJANGO_CONSTRAINTS_TO_RECREATE = [
+    ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_371b52d98485a593_uniq", "UNIQUE (mailinglist_id, thread_id)"),
+    ("hyperkitty_thread", "mailinglist_id_refs_name_3725eec2", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
+    ("hyperkitty_email", "hyperkitty_email_mailinglist_id_57f04aace3f8ee5e_uniq", "UNIQUE (mailinglist_id, message_id)"),
+    ("hyperkitty_email", "mailinglist_id_refs_name_654506d3", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
+]
 
-def update_col(connection, table, column, where=None, pk="id"):
+
+def get_mapping(cursor, table, column):
+    ml_mapping = {}
+    query = "SELECT {c} FROM {t}".format(c=column, t=table)
+    print(query)
+    cursor.execute(query)
+    for row in cursor:
+        value = row[0]
+        orig_value = value.replace(
+            "lists.stg.fedoraproject.org", "lists.fedoraproject.org"
+            ).replace(
+            "lists.stg.fedorahosted.org", "lists.fedorahosted.org")
+        changed_value = value.replace(
+            "lists.fedoraproject.org", "lists.stg.fedoraproject.org"
+            ).replace(
+            "lists.fedorahosted.org", "lists.stg.fedorahosted.org")
+        if orig_value == changed_value:
+            continue
+        ml_mapping[orig_value] = changed_value
+    return ml_mapping
+
+
+def update_col_1(connection, table, column, where=None, pk="id"):
     cursor = connection.cursor()
     cursor_2 = connection.cursor()
-    query = "SELECT {pk}, {c} FROM {t}".format(t=table, c=column, pk=pk)
+    where = " WHERE {}".format(where) if where is not None else ""
-    if where:
+    #query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
-        query += " WHERE {}".format(where)
+    #cursor.execute(query)
+    #count = cursor.fetchone()[0]
+    query = "SELECT {pk}, {c} FROM {t} {w}".format(
+        t=table, c=column, pk=pk, w=where)
     #query += " LIMIT 10000"
+    print(query)
+    #print("{} lines".format(count))
+    updates = []
     cursor.execute(query)
     print(cursor.query)
-    updates = []
     for record_id, value in cursor:
         changed_value = value.replace(
             "lists.fedoraproject.org", "lists.stg.fedoraproject.org"
@@ -61,20 +100,30 @@ def update_col(connection, table, column, where=None, pk="id"):
             cursor_2.execute(
                 "SELECT 1 FROM {t} WHERE {pk} = %s".format(t=table, pk=pk),
                 [changed_value])
-            #result = cursor_2.fetchone()
-            #print(repr(result), updates, value, changed_value)
-            #IF result:
             if cursor_2.fetchone():
                 print("Skipping {v} in {t}".format(t=table, v=changed_value))
                 continue
         updates.append([changed_value, record_id])
     cursor_2.close()
-    if updates:
+    print("Doing {} updates now".format(len(updates)))
-        query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format(
+    query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format(
-            t=table, c=column, pk=pk)
+        t=table, c=column, pk=pk)
-        print(query, "with %d params" % len(updates))
+    print(query, "with %d params" % len(updates))
-        cursor.executemany(query, updates)
+    cursor.executemany(query, updates)
-    cursor.close()
+
+
+def update_col_2(ml_mapping, cursor, table, column, where=None):
+    query_where = " AND {}".format(where) if where is not None else ""
+    query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
+    cursor.execute(query)
+    count = cursor.fetchone()[0]
+    print("Updating {} rows.".format(count))
+    for name_orig, name_new in ml_mapping.items():
+        query = "UPDATE {t} SET {c} = %s WHERE {c} = %s {w}".format(
+            t=table, c=column, w=query_where)
+        params = (name_new, name_orig)
+        print(query % params)
+        cursor.execute(query, params)
 
 
 def do_mailman():
@@ -86,49 +135,67 @@ def do_mailman():
     #    "dbname={scheme} user={username} password={password} host={hostname}".format(db_url)
     #    )
 
-    for table, column in MAILMAN_TABLES_TO_REPLACE:
+    with conn.cursor() as cursor:
-        update_col(conn, table, column)
+        ml_mapping = get_mapping(cursor, "mailinglist", "list_id")
-    update_col(conn, "pendedkeyvalue", "value",
+        for table, column in MAILMAN_TABLES_TO_REPLACE:
-               """ "key" = 'list_id' OR "key" = '_mod_listid'
+            update_col_1(conn, table, column)
-                  OR "key" = 'envsender'""")
+        update_col_2(ml_mapping, cursor, "ban", "list_id")
+        update_col_2(ml_mapping, cursor, "member", "list_id")
+        update_col_2(ml_mapping, cursor, "bounceevent", "list_id")
+        update_col_1(conn, "pendedkeyvalue", "value",
+                     """ "key" = 'list_id' OR "key" = '_mod_listid' """
+                     """ OR "key" = 'envsender'""")
 
-    cursor = conn.cursor()
+        cursor.execute("UPDATE \"user\" SET password = 'INVALID'")
-    cursor.execute("UPDATE \"user\" SET password = 'INVALID'")
+        print(cursor.query)
-    print(cursor.query)
+        cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE")
-    cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE")
+        print(cursor.query)
-    print(cursor.query)
-    cursor.close()
     conn.commit()
     conn.close()
     call(["sudo", "-u", "mailman", "mailman3", "aliases"])
 
+
 def do_django():
     from django.db import connection, transaction
     from django.core.management import call_command
-    cursor = connection.cursor()
+    with connection.cursor() as cursor:
-    cursor.execute("UPDATE auth_user SET password = '!INVALID'")
+        cursor.execute("UPDATE auth_user SET password = '!INVALID'")
-    print(cursor.query)
-    # Empty tables that contain sensitive data
-    for table in DJANGO_TABLES_TO_EMPTY:
-        cursor.execute("DELETE FROM %s" % table)
         print(cursor.query)
-    with transaction.atomic():
+        # Empty tables that contain sensitive data
-        cursor.execute("SET CONSTRAINTS ALL DEFERRED")
+        for table in DJANGO_TABLES_TO_EMPTY:
-        # Replace in tables with prod domains:
+            cursor.execute("DELETE FROM %s" % table)
-        update_col(connection, "django_mailman3_maildomain", "mail_domain")
+            print(cursor.query)
-        update_col(connection, "hyperkitty_mailinglist", "name", pk="name")
+        #for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
-        cursor.execute("select name from hyperkitty_mailinglist order by name")
+        #    cursor.execute(
-        for row in cursor:
+        #        "ALTER TABLE {t} DROP CONSTRAINT IF EXISTS {n}".format(
-            print(row[0])
+        #            t=table, n=name))
-        update_col(connection, "hyperkitty_thread", "mailinglist_id")
+        #    print(cursor.query)
-        update_col(connection, "hyperkitty_email", "mailinglist_id")
+        for table, name, column in DJANGO_INDICES_TO_RECREATE:
-        cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+            cursor.execute("DROP INDEX IF EXISTS {n}".format(t=table, n=name))
-    cursor.close()
+            print(cursor.query)
+        with transaction.atomic():
+            cursor.execute("SET CONSTRAINTS ALL DEFERRED")
+            ml_mapping = get_mapping(cursor, "hyperkitty_mailinglist", "name")
+            # Replace in tables with prod domains:
+            update_col_1(connection, "django_mailman3_maildomain", "mail_domain")
+            update_col_1(connection, "hyperkitty_mailinglist", "name", pk="name")
+            update_col_2(ml_mapping, cursor, "hyperkitty_thread", "mailinglist_id")
+            update_col_2(ml_mapping, cursor, "hyperkitty_email", "mailinglist_id")
+            cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
+        for table, name, column in DJANGO_INDICES_TO_RECREATE:
+            cursor.execute("CREATE INDEX {n} ON {t} {c}".format(
+                           n=name, t=table, c=column))
+            print(cursor.query)
+        #for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
+        #    cursor.execute("ALTER TABLE {t} ADD CONSTRAINT {n} {c}".format(
+        #                   n=name, t=table, c=create))
+        #    print(cursor.query)
     connection.commit()
     call_command("loaddata", "/srv/webui/config/initial-data.json")
 
 
 def main():
+    call(["systemctl", "stop", "webui-qcluster"])
     call(["systemctl", "stop", "mailman3"])
     call(["systemctl", "stop", "httpd"])
     call(["systemctl", "stop", "crond"])
@@ -137,6 +204,7 @@ def main():
     call(["systemctl", "start", "crond"])
     call(["systemctl", "start", "httpd"])
     call(["systemctl", "start", "mailman3"])
+    call(["systemctl", "start", "webui-qcluster"])
 
 
 if __name__ == "__main__":

roles/mailman/tasks/main.yml

@@ -412,6 +412,13 @@
   - import-mm2.py
   - periodic.py
 
+- name: install the templatized scripts
+  template: src={{ item }}.j2 dest="{{ mailman_webui_basedir }}/bin/{{ item }}"
+            owner=root group=root mode=0755
+  tags: mailman
+  with_items:
+  - manage.py
+
 - name: install the staging-sync script
   copy: src=prod-to-stg.py dest="{{ mailman_webui_basedir }}/bin/prod-to-stg.py"
   when: env == "staging"

roles/mailman/templates/manage.py.j2

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if __name__ == "__main__":
+    sys.path.insert(0, "{{ mailman_webui_confdir }}")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+    from django.core.management import execute_from_command_line
+    execute_from_command_line(sys.argv)