From abee63c966db8a62d4e6e37941d2b2f6aec73644 Mon Sep 17 00:00:00 2001
From: Kevin Fenzi <kevin@scrye.com>
Date: Mon, 20 May 2024 10:28:04 -0700
Subject: [PATCH] postgresql_server: adjust backups for db-datanommer02

Currently backups are taking 17-18 hours with 4 threads.
Now that we have 16 cpus defined there, lets bump that up to 8 and see
if that lowers things much. If not we can look at moving to another
compression, but the database is very large so lots of compression is
good to save disk space.

Also filter out another output of the backup job that causes cron
emails.

Signed-off-by: Kevin Fenzi <kevin@scrye.com>
---
 .../files/backup-database.db-datanommer02        | 16 ++++++++++++++++
 roles/postgresql_server/tasks/main.yml           | 12 +++++++++---
 .../templates/cron-backup-database               |  2 +-
 3 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 roles/postgresql_server/files/backup-database.db-datanommer02

diff --git a/roles/postgresql_server/files/backup-database.db-datanommer02 b/roles/postgresql_server/files/backup-database.db-datanommer02
new file mode 100644
index 0000000000..511cd0ad3a
--- /dev/null
+++ b/roles/postgresql_server/files/backup-database.db-datanommer02
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Backup a database *locally* to /backups/.
+
+# Sleep a bit so we do not have a thundering herd on db hosts
+sleep $[ ( $RANDOM % 7200 )  + 1 ]s
+
+DB=$1
+
+# Make our latest backup
+# Make it use a limited number of threads because pxz will use all the
+# cpus which causes pg_dump to starve which causes...
+
+/usr/bin/pg_dump --exclude-table-data=sessions -C $DB | /usr/bin/pxz -T8 > /backups/$DB-$(date +%F).dump.xz
+
+# Also, delete the backup from a few days ago.
+rm -f /backups/$DB-$(date --date="1 days ago" +%F).dump.xz
diff --git a/roles/postgresql_server/tasks/main.yml b/roles/postgresql_server/tasks/main.yml
index d159c208e0..4d2612a4bc 100644
--- a/roles/postgresql_server/tasks/main.yml
+++ b/roles/postgresql_server/tasks/main.yml
@@ -138,7 +138,7 @@
 
 - name: Copy over backup scriplet
   copy: src=backup-database dest=/usr/local/bin/backup-database mode=0755
-  when: not inventory_hostname.startswith('db-koji01.iad2')
+  when: not inventory_hostname.startswith(('db-koji01.iad2','db-datanommer'))
   tags:
   - postgresql
 
@@ -148,17 +148,23 @@
   tags:
   - postgresql
 
+- name: Copy over backup scriplet
+  copy: src=backup-database.db-datanommer02 dest=/usr/local/bin/backup-database mode=0755
+  when: inventory_hostname.startswith('db-datanommer02.iad2')
+  tags:
+  - postgresql
+
 - name: Copy over anitya public backup scriplet
   copy: src=backup-database.anitya dest=/usr/local/bin/backup-database.anitya mode=0755
   tags:
   - postgresql
   when: inventory_hostname.startswith('db01.phx2') or inventory_hostname.startswith('db01.iad2')
 
-- name: Copy over anitya publib backup cron
+- name: Copy over anitya public backup cron
   copy: src=cron-backup-anitya-public dest=/etc/cron.d/cron-backup-anitya-public mode=0644
   tags:
   - postgresql
-  when: inventory_hostname.startswith('db01.phx2') or inventory_hostname.startswith('db01.iad2')
+  when: inventory_hostname.startswith('db01.iad2')
 
 - name: Set up some cronjobs to backup databases as configured
   template: >
diff --git a/roles/postgresql_server/templates/cron-backup-database b/roles/postgresql_server/templates/cron-backup-database
index 24ef3de6b0..777d6e370b 100644
--- a/roles/postgresql_server/templates/cron-backup-database
+++ b/roles/postgresql_server/templates/cron-backup-database
@@ -1,5 +1,5 @@
 {% if inventory_hostname == 'db-datanommer02.iad2.fedoraproject.org' %}
-0 0 * * * postgres /usr/local/bin/backup-database {{ item }} |& grep -Ev 'warning:|hypertable|chunk|restore|data-only'
+0 0 * * * postgres /usr/local/bin/backup-database {{ item }} |& grep -Ev 'warning:|hypertable|chunk|restore|data-only|continuous_agg'
 {% else %}
 0 0 * * * postgres /usr/local/bin/backup-database {{ item }}
 {% endif %}