diff --git a/scripts/migration_timescaledb.sql b/scripts/migration_timescaledb.sql index c03f5c0..93c8b05 100644 --- a/scripts/migration_timescaledb.sql +++ b/scripts/migration_timescaledb.sql @@ -120,3 +120,27 @@ CREATE INDEX messages2_datanommer_timestamp_topic_idx ON public.messages2 USING -- ALTER TABLE package_messages2 -- ADD CONSTRAINT package_messages2_msg_fkey -- FOREIGN KEY (msg, "timestamp") REFERENCES messages2(id, "timestamp") MATCH FULL; + + + +-- Save some SQL queries we've used so we can find them back later if needed/desired + + +-- Find duplicates with the same msg_id/timestamp in messages2 +-- Restrict to messages since January 1st 2021 as we know the dups are recent +select msg_id, "timestamp", count(msg_id) as cnt + from messages2 + where "timestamp" > TO_TIMESTAMP('2021-01-01 01:00:00', 'YYYY-MM-DD HH:MI:SS') + group by msg_id, timestamp + having count(msg_id) > 1; + +-- Remove the duplicates messages with the same msg_id/timestamp in messages2 +DELETE FROM + messages2 m1 + USING messages2 m2 +WHERE + m1.id < m2.id + AND m1."timestamp" = m2."timestamp" + AND m1.msg_id = m2.msg_id + AND m1."timestamp" > TO_TIMESTAMP('2021-01-01 01:00:00', 'YYYY-MM-DD HH:MI:SS') + AND m2."timestamp" > TO_TIMESTAMP('2021-01-01 01:00:00', 'YYYY-MM-DD HH:MI:SS');