Mailman: improve staging-sync script, add manage.py
This commit is contained in:
parent
ac9f8f3d66
commit
f4a1d1ede8
4 changed files with 133 additions and 47 deletions
|
@ -39,4 +39,5 @@ echo "unit tests"
|
|||
django-admin test --pythonpath $CONFDIR --settings settings_test django_mailman3 hyperkitty postorius
|
||||
|
||||
# Restart services
|
||||
echo "Start services"
|
||||
systemctl start httpd mailman3 crond webui-qcluster
|
||||
|
|
|
@ -11,7 +11,7 @@ from subprocess import call
|
|||
from urlparse import urlparse
|
||||
|
||||
if not os.getenv("DJANGO_SETTINGS_MODULE"):
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "settings"
|
||||
os.environ["DJANGO_SETTINGS_MODULE"] = "settings_admin"
|
||||
sys.path.insert(0, "/srv/webui/config")
|
||||
|
||||
import django
|
||||
|
@ -22,9 +22,9 @@ MAILMAN_TABLES_TO_REPLACE = [
|
|||
("domain", "mail_host"),
|
||||
("mailinglist", "mail_host"),
|
||||
("mailinglist", "list_id"),
|
||||
("ban", "list_id"),
|
||||
("bounceevent", "list_id"),
|
||||
("member", "list_id"),
|
||||
#("ban", "list_id"),
|
||||
#("bounceevent", "list_id"),
|
||||
#("member", "list_id"),
|
||||
("template", "context"),
|
||||
("acceptablealias", "alias"),
|
||||
]
|
||||
|
@ -37,17 +37,56 @@ DJANGO_TABLES_TO_EMPTY = [
|
|||
"social_auth_usersocialauth",
|
||||
"socialaccount_socialtoken",
|
||||
]
|
||||
DJANGO_INDICES_TO_RECREATE = [
|
||||
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id", "(mailinglist_id)"),
|
||||
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
|
||||
("hyperkitty_email", "hyperkitty_email_mailinglist_id", "(mailinglist_id)"),
|
||||
("hyperkitty_email", "hyperkitty_email_mailinglist_id_like", "(mailinglist_id varchar_pattern_ops)"),
|
||||
]
|
||||
DJANGO_CONSTRAINTS_TO_RECREATE = [
|
||||
("hyperkitty_thread", "hyperkitty_thread_mailinglist_id_371b52d98485a593_uniq", "UNIQUE (mailinglist_id, thread_id)"),
|
||||
("hyperkitty_thread", "mailinglist_id_refs_name_3725eec2", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
|
||||
("hyperkitty_email", "hyperkitty_email_mailinglist_id_57f04aace3f8ee5e_uniq", "UNIQUE (mailinglist_id, message_id)"),
|
||||
("hyperkitty_email", "mailinglist_id_refs_name_654506d3", "FOREIGN KEY (mailinglist_id) REFERENCES hyperkitty_mailinglist(name) DEFERRABLE INITIALLY DEFERRED"),
|
||||
]
|
||||
|
||||
def update_col(connection, table, column, where=None, pk="id"):
|
||||
|
||||
def get_mapping(cursor, table, column):
|
||||
ml_mapping = {}
|
||||
query = "SELECT {c} FROM {t}".format(c=column, t=table)
|
||||
print(query)
|
||||
cursor.execute(query)
|
||||
for row in cursor:
|
||||
value = row[0]
|
||||
orig_value = value.replace(
|
||||
"lists.stg.fedoraproject.org", "lists.fedoraproject.org"
|
||||
).replace(
|
||||
"lists.stg.fedorahosted.org", "lists.fedorahosted.org")
|
||||
changed_value = value.replace(
|
||||
"lists.fedoraproject.org", "lists.stg.fedoraproject.org"
|
||||
).replace(
|
||||
"lists.fedorahosted.org", "lists.stg.fedorahosted.org")
|
||||
if orig_value == changed_value:
|
||||
continue
|
||||
ml_mapping[orig_value] = changed_value
|
||||
return ml_mapping
|
||||
|
||||
|
||||
def update_col_1(connection, table, column, where=None, pk="id"):
|
||||
cursor = connection.cursor()
|
||||
cursor_2 = connection.cursor()
|
||||
query = "SELECT {pk}, {c} FROM {t}".format(t=table, c=column, pk=pk)
|
||||
if where:
|
||||
query += " WHERE {}".format(where)
|
||||
where = " WHERE {}".format(where) if where is not None else ""
|
||||
#query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
|
||||
#cursor.execute(query)
|
||||
#count = cursor.fetchone()[0]
|
||||
query = "SELECT {pk}, {c} FROM {t} {w}".format(
|
||||
t=table, c=column, pk=pk, w=where)
|
||||
#query += " LIMIT 10000"
|
||||
print(query)
|
||||
#print("{} lines".format(count))
|
||||
updates = []
|
||||
cursor.execute(query)
|
||||
print(cursor.query)
|
||||
updates = []
|
||||
for record_id, value in cursor:
|
||||
changed_value = value.replace(
|
||||
"lists.fedoraproject.org", "lists.stg.fedoraproject.org"
|
||||
|
@ -61,20 +100,30 @@ def update_col(connection, table, column, where=None, pk="id"):
|
|||
cursor_2.execute(
|
||||
"SELECT 1 FROM {t} WHERE {pk} = %s".format(t=table, pk=pk),
|
||||
[changed_value])
|
||||
#result = cursor_2.fetchone()
|
||||
#print(repr(result), updates, value, changed_value)
|
||||
#IF result:
|
||||
if cursor_2.fetchone():
|
||||
print("Skipping {v} in {t}".format(t=table, v=changed_value))
|
||||
continue
|
||||
updates.append([changed_value, record_id])
|
||||
cursor_2.close()
|
||||
if updates:
|
||||
query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format(
|
||||
t=table, c=column, pk=pk)
|
||||
print(query, "with %d params" % len(updates))
|
||||
cursor.executemany(query, updates)
|
||||
cursor.close()
|
||||
print("Doing {} updates now".format(len(updates)))
|
||||
query = "UPDATE {t} SET {c} = %s WHERE {pk} = %s".format(
|
||||
t=table, c=column, pk=pk)
|
||||
print(query, "with %d params" % len(updates))
|
||||
cursor.executemany(query, updates)
|
||||
|
||||
|
||||
def update_col_2(ml_mapping, cursor, table, column, where=None):
|
||||
query_where = " AND {}".format(where) if where is not None else ""
|
||||
query = "SELECT COUNT(*) FROM {t} {w}".format(t=table, w=where)
|
||||
cursor.execute(query)
|
||||
count = cursor.fetchone()[0]
|
||||
print("Updating {} rows.".format(count))
|
||||
for name_orig, name_new in ml_mapping.items():
|
||||
query = "UPDATE {t} SET {c} = %s WHERE {c} = %s {w}".format(
|
||||
t=table, c=column, w=query_where)
|
||||
params = (name_new, name_orig)
|
||||
print(query % params)
|
||||
cursor.execute(query, params)
|
||||
|
||||
|
||||
def do_mailman():
|
||||
|
@ -86,49 +135,67 @@ def do_mailman():
|
|||
# "dbname={scheme} user={username} password={password} host={hostname}".format(db_url)
|
||||
# )
|
||||
|
||||
for table, column in MAILMAN_TABLES_TO_REPLACE:
|
||||
update_col(conn, table, column)
|
||||
update_col(conn, "pendedkeyvalue", "value",
|
||||
""" "key" = 'list_id' OR "key" = '_mod_listid'
|
||||
OR "key" = 'envsender'""")
|
||||
with conn.cursor() as cursor:
|
||||
ml_mapping = get_mapping(cursor, "mailinglist", "list_id")
|
||||
for table, column in MAILMAN_TABLES_TO_REPLACE:
|
||||
update_col_1(conn, table, column)
|
||||
update_col_2(ml_mapping, cursor, "ban", "list_id")
|
||||
update_col_2(ml_mapping, cursor, "member", "list_id")
|
||||
update_col_2(ml_mapping, cursor, "bounceevent", "list_id")
|
||||
update_col_1(conn, "pendedkeyvalue", "value",
|
||||
""" "key" = 'list_id' OR "key" = '_mod_listid' """
|
||||
""" OR "key" = 'envsender'""")
|
||||
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("UPDATE \"user\" SET password = 'INVALID'")
|
||||
print(cursor.query)
|
||||
cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE")
|
||||
print(cursor.query)
|
||||
cursor.close()
|
||||
cursor.execute("UPDATE \"user\" SET password = 'INVALID'")
|
||||
print(cursor.query)
|
||||
cursor.execute("UPDATE \"mailinglist\" SET digests_enabled = FALSE")
|
||||
print(cursor.query)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
call(["sudo", "-u", "mailman", "mailman3", "aliases"])
|
||||
|
||||
|
||||
def do_django():
|
||||
from django.db import connection, transaction
|
||||
from django.core.management import call_command
|
||||
cursor = connection.cursor()
|
||||
cursor.execute("UPDATE auth_user SET password = '!INVALID'")
|
||||
print(cursor.query)
|
||||
# Empty tables that contain sensitive data
|
||||
for table in DJANGO_TABLES_TO_EMPTY:
|
||||
cursor.execute("DELETE FROM %s" % table)
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("UPDATE auth_user SET password = '!INVALID'")
|
||||
print(cursor.query)
|
||||
with transaction.atomic():
|
||||
cursor.execute("SET CONSTRAINTS ALL DEFERRED")
|
||||
# Replace in tables with prod domains:
|
||||
update_col(connection, "django_mailman3_maildomain", "mail_domain")
|
||||
update_col(connection, "hyperkitty_mailinglist", "name", pk="name")
|
||||
cursor.execute("select name from hyperkitty_mailinglist order by name")
|
||||
for row in cursor:
|
||||
print(row[0])
|
||||
update_col(connection, "hyperkitty_thread", "mailinglist_id")
|
||||
update_col(connection, "hyperkitty_email", "mailinglist_id")
|
||||
cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
|
||||
cursor.close()
|
||||
# Empty tables that contain sensitive data
|
||||
for table in DJANGO_TABLES_TO_EMPTY:
|
||||
cursor.execute("DELETE FROM %s" % table)
|
||||
print(cursor.query)
|
||||
#for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
|
||||
# cursor.execute(
|
||||
# "ALTER TABLE {t} DROP CONSTRAINT IF EXISTS {n}".format(
|
||||
# t=table, n=name))
|
||||
# print(cursor.query)
|
||||
for table, name, column in DJANGO_INDICES_TO_RECREATE:
|
||||
cursor.execute("DROP INDEX IF EXISTS {n}".format(t=table, n=name))
|
||||
print(cursor.query)
|
||||
with transaction.atomic():
|
||||
cursor.execute("SET CONSTRAINTS ALL DEFERRED")
|
||||
ml_mapping = get_mapping(cursor, "hyperkitty_mailinglist", "name")
|
||||
# Replace in tables with prod domains:
|
||||
update_col_1(connection, "django_mailman3_maildomain", "mail_domain")
|
||||
update_col_1(connection, "hyperkitty_mailinglist", "name", pk="name")
|
||||
update_col_2(ml_mapping, cursor, "hyperkitty_thread", "mailinglist_id")
|
||||
update_col_2(ml_mapping, cursor, "hyperkitty_email", "mailinglist_id")
|
||||
cursor.execute("SET CONSTRAINTS ALL IMMEDIATE")
|
||||
for table, name, column in DJANGO_INDICES_TO_RECREATE:
|
||||
cursor.execute("CREATE INDEX {n} ON {t} {c}".format(
|
||||
n=name, t=table, c=column))
|
||||
print(cursor.query)
|
||||
#for table, name, create in DJANGO_CONSTRAINTS_TO_RECREATE:
|
||||
# cursor.execute("ALTER TABLE {t} ADD CONSTRAINT {n} {c}".format(
|
||||
# n=name, t=table, c=create))
|
||||
# print(cursor.query)
|
||||
connection.commit()
|
||||
call_command("loaddata", "/srv/webui/config/initial-data.json")
|
||||
|
||||
|
||||
def main():
|
||||
call(["systemctl", "stop", "webui-qcluster"])
|
||||
call(["systemctl", "stop", "mailman3"])
|
||||
call(["systemctl", "stop", "httpd"])
|
||||
call(["systemctl", "stop", "crond"])
|
||||
|
@ -137,6 +204,7 @@ def main():
|
|||
call(["systemctl", "start", "crond"])
|
||||
call(["systemctl", "start", "httpd"])
|
||||
call(["systemctl", "start", "mailman3"])
|
||||
call(["systemctl", "start", "webui-qcluster"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue