From f4a1d1ede8d5fce0983b693df9e9ef274de14309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Sun, 25 Jun 2017 08:42:33 +0000 Subject: [PATCH] Mailman: improve staging-sync script, add manage.py --- roles/mailman/files/post-update.sh | 1 + roles/mailman/files/prod-to-stg.py | 162 +++++++++++++++++++-------- roles/mailman/tasks/main.yml | 7 ++ roles/mailman/templates/manage.py.j2 | 10 ++ 4 files changed, 133 insertions(+), 47 deletions(-) create mode 100644 roles/mailman/templates/manage.py.j2 diff --git a/roles/mailman/files/post-update.sh b/roles/mailman/files/post-update.sh index 0b95d0a618..c8ab94e51e 100755 --- a/roles/mailman/files/post-update.sh +++ b/roles/mailman/files/post-update.sh @@ -39,4 +39,5 @@ echo "unit tests" django-admin test --pythonpath $CONFDIR --settings settings_test django_mailman3 hyperkitty postorius # Restart services +echo "Start services" systemctl start httpd mailman3 crond webui-qcluster diff --git a/roles/mailman/files/prod-to-stg.py b/roles/mailman/files/prod-to-stg.py index 1db327b565..fc3356cf96 100755 --- a/roles/mailman/files/prod-to-stg.py +++ b/roles/mailman/files/prod-to-stg.py @@ -11,7 +11,7 @@ from subprocess import call from urlparse import urlparse if not os.getenv("DJANGO_SETTINGS_MODULE"): - os.environ["DJANGO_SETTINGS_MODULE"] = "settings" + os.environ["DJANGO_SETTINGS_MODULE"] = "settings_admin" sys.path.insert(0, "/srv/webui/config") import django @@ -22,9 +22,9 @@ MAILMAN_TABLES_TO_REPLACE = [ ("domain", "mail_host"), ("mailinglist", "mail_host"), ("mailinglist", "list_id"), - ("ban", "list_id"), - ("bounceevent", "list_id"), - ("member", "list_id"), + #("ban", "list_id"), + #("bounceevent", "list_id"), + #("member", "list_id"), ("template", "context"), ("acceptablealias", "alias"), ] @@ -37,17 +37,56 @@ DJANGO_TABLES_TO_EMPTY = [ "social_auth_usersocialauth", "socialaccount_socialtoken", ] +DJANGO_INDICES_TO_RECREATE = [ + ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id", "(mailinglist_id)"), + ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"), + ("hyperkitty_email", "hyperkitty_email_mailinglist_id", "(mailinglist_id)"), + ("hyperkitty_email", "hyperkitty_email_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"), +] +DJANGO_CONSTRAINTS_TO_RECREATE = [ + ("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_371b52d98485a593_uniq", "UNIQUE (mailinglist_id, thread_id)"), + ("hyperkitty_thread", "mailinglist_id_refs_name_3725eec2", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"), + ("hyperkitty_email", "hyperkitty_email_mailinglist_id_57f04aace3f8ee5e_uniq", "UNIQUE (mailinglist_id, message_id)"), + ("hyperkitty_email", "mailinglist_id_refs_name_654506d3", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"), +] -def update_col(connection, table, column, where=None, pk="id"): + +def get_mapping(cursor, table, column): + ml_mapping = {} + query = "SELECT {c} FROM {t}".format(c=column, t=table) + print(query) + cursor.execute(query) + for row in cursor: + value = row[0] + orig_value = value.replace( + "lists.stg.fedoraproject.org", "lists.fedoraproject.org" + ).replace( + "lists.stg.fedorahosted.org", "lists.fedorahosted.org") + changed_value = value.replace( + "lists.fedoraproject.org", "lists.stg.fedoraproject.org" + ).replace( + "lists.fedorahosted.org", "lists.stg.fedorahosted.org") + if orig_value == changed_value: + continue + ml_mapping[orig_value] = changed_value + return ml_mapping + + +def update_col_1(connection, table, column, where=None, pk="id"): cursor = connection.cursor() cursor_2 = connection.cursor() - query = "SELECT {pk}, {c} FROM {t}".format(t=table, c=column, pk=pk) - if where: - query += " WHERE {}".format(where) + where = " WHERE {}".format(where) if where is not None else "" + #query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where) + #cursor.execute(query) + #count = cursor.fetchone()[0] + query = "SELECT {pk}, {c} FROM {t} {w}".format( + t=table, c=column, pk=pk, w=where) #query += " LIMIT 10000" + print(query) + #print("{} lines".format(count)) + updates = [] cursor.execute(query) print(cursor.query) - updates = [] for record_id, value in cursor: changed_value = value.replace( "lists.fedoraproject.org", "lists.stg.fedoraproject.org" @@ -61,20 +100,30 @@ def update_col(connection, table, column, where=None, pk="id"): cursor_2.execute( "SELECT 1 FROM {t} WHERE {pk} = %s".format(t=table, pk=pk), [changed_value]) - #result = cursor_2.fetchone() - #print(repr(result), updates, value, changed_value) - #IF result: if cursor_2.fetchone(): print("Skipping {v} in {t}".format(t=table, v=changed_value)) continue updates.append([changed_value, record_id]) cursor_2.close() - if updates: - query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format( - t=table, c=column, pk=pk) - print(query, "with %d params" % len(updates)) - cursor.executemany(query, updates) - cursor.close() + print("Doing {} updates now".format(len(updates))) + query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format( + t=table, c=column, pk=pk) + print(query, "with %d params" % len(updates)) + cursor.executemany(query, updates) + + +def update_col_2(ml_mapping, cursor, table, column, where=None): + query_where = " AND {}".format(where) if where is not None else "" + query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where) + cursor.execute(query) + count = cursor.fetchone()[0] + print("Updating {} rows.".format(count)) + for name_orig, name_new in ml_mapping.items(): + query = "UPDATE {t} SET {c} = %s WHERE {c} = %s {w}".format( + t=table, c=column, w=query_where) + params = (name_new, name_orig) + print(query % params) + cursor.execute(query, params) def do_mailman(): @@ -86,49 +135,67 @@ def do_mailman(): # "dbname={scheme} user={username} password={password} host={hostname}".format(db_url) # ) - for table, column in MAILMAN_TABLES_TO_REPLACE: - update_col(conn, table, column) - update_col(conn, "pendedkeyvalue", "value", - """ "key" = 'list_id' OR "key" = '_mod_listid' - OR "key" = 'envsender'""") + with conn.cursor() as cursor: + ml_mapping = get_mapping(cursor, "mailinglist", "list_id") + for table, column in MAILMAN_TABLES_TO_REPLACE: + update_col_1(conn, table, column) + update_col_2(ml_mapping, cursor, "ban", "list_id") + update_col_2(ml_mapping, cursor, "member", "list_id") + update_col_2(ml_mapping, cursor, "bounceevent", "list_id") + update_col_1(conn, "pendedkeyvalue", "value", + """ "key" = 'list_id' OR "key" = '_mod_listid' """ + """ OR "key" = 'envsender'""") - cursor = conn.cursor() - cursor.execute("UPDATE \"user\" SET password = 'INVALID'") - print(cursor.query) - cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE") - print(cursor.query) - cursor.close() + cursor.execute("UPDATE \"user\" SET password = 'INVALID'") + print(cursor.query) + cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE") + print(cursor.query) conn.commit() conn.close() call(["sudo", "-u", "mailman", "mailman3", "aliases"]) + def do_django(): from django.db import connection, transaction from django.core.management import call_command - cursor = connection.cursor() - cursor.execute("UPDATE auth_user SET password = '!INVALID'") - print(cursor.query) - # Empty tables that contain sensitive data - for table in DJANGO_TABLES_TO_EMPTY: - cursor.execute("DELETE FROM %s" % table) + with connection.cursor() as cursor: + cursor.execute("UPDATE auth_user SET password = '!INVALID'") print(cursor.query) - with transaction.atomic(): - cursor.execute("SET CONSTRAINTS ALL DEFERRED") - # Replace in tables with prod domains: - update_col(connection, "django_mailman3_maildomain", "mail_domain") - update_col(connection, "hyperkitty_mailinglist", "name", pk="name") - cursor.execute("select name from hyperkitty_mailinglist order by name") - for row in cursor: - print(row[0]) - update_col(connection, "hyperkitty_thread", "mailinglist_id") - update_col(connection, "hyperkitty_email", "mailinglist_id") - cursor.execute("SET CONSTRAINTS ALL IMMEDIATE") - cursor.close() + # Empty tables that contain sensitive data + for table in DJANGO_TABLES_TO_EMPTY: + cursor.execute("DELETE FROM %s" % table) + print(cursor.query) + #for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE: + # cursor.execute( + # "ALTER TABLE {t} DROP CONSTRAINT IF EXISTS {n}".format( + # t=table, n=name)) + # print(cursor.query) + for table, name, column in DJANGO_INDICES_TO_RECREATE: + cursor.execute("DROP INDEX IF EXISTS {n}".format(t=table, n=name)) + print(cursor.query) + with transaction.atomic(): + cursor.execute("SET CONSTRAINTS ALL DEFERRED") + ml_mapping = get_mapping(cursor, "hyperkitty_mailinglist", "name") + # Replace in tables with prod domains: + update_col_1(connection, "django_mailman3_maildomain", "mail_domain") + update_col_1(connection, "hyperkitty_mailinglist", "name", pk="name") + update_col_2(ml_mapping, cursor, "hyperkitty_thread", "mailinglist_id") + update_col_2(ml_mapping, cursor, "hyperkitty_email", "mailinglist_id") + cursor.execute("SET CONSTRAINTS ALL IMMEDIATE") + for table, name, column in DJANGO_INDICES_TO_RECREATE: + cursor.execute("CREATE INDEX {n} ON {t} {c}".format( + n=name, t=table, c=column)) + print(cursor.query) + #for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE: + # cursor.execute("ALTER TABLE {t} ADD CONSTRAINT {n} {c}".format( + # n=name, t=table, c=create)) + # print(cursor.query) connection.commit() call_command("loaddata", "/srv/webui/config/initial-data.json") def main(): + call(["systemctl", "stop", "webui-qcluster"]) call(["systemctl", "stop", "mailman3"]) call(["systemctl", "stop", "httpd"]) call(["systemctl", "stop", "crond"]) @@ -137,6 +204,7 @@ def main(): call(["systemctl", "start", "crond"]) call(["systemctl", "start", "httpd"]) call(["systemctl", "start", "mailman3"]) + call(["systemctl", "start", "webui-qcluster"]) if __name__ == "__main__": diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml index 8ba14a361f..532277e284 100644 --- a/roles/mailman/tasks/main.yml +++ b/roles/mailman/tasks/main.yml @@ -412,6 +412,13 @@ - import-mm2.py - periodic.py +- name: install the templatized scripts + template: src={{ item }}.j2 dest="{{ mailman_webui_basedir }}/bin/{{ item }}" + owner=root group=root mode=0755 + tags: mailman + with_items: + - manage.py + - name: install the staging-sync script copy: src=prod-to-stg.py dest="{{ mailman_webui_basedir }}/bin/prod-to-stg.py" when: env == "staging" diff --git a/roles/mailman/templates/manage.py.j2 b/roles/mailman/templates/manage.py.j2 new file mode 100644 index 0000000000..cb11412e80 --- /dev/null +++ b/roles/mailman/templates/manage.py.j2 @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +import os +import sys + +if __name__ == "__main__": + sys.path.insert(0, "{{ mailman_webui_confdir }}") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") + from django.core.management import execute_from_command_line + execute_from_command_line(sys.argv)