copr: incremental backup to storinator, part 1

These scripts are based on my personal "Don't Delay Backups" project,
which is not yet available as a public role.

inventory/group_vars/copr_aws

@@ -75,3 +75,13 @@ root_auth_users: msuchy frostyx praiskup nikromen
 aws_cloudfront_distribution: E2PUZIRCXCOXTG
 
 nrpe_client_uid: 500
+
+rsnapshot_push:
+  server_host: storinator01.rdu-cc.fedoraproject.org
+  backup_dir: /srv/nfs/copr-be
+  timing_plan: copr_be
+  cases:
+    copr-be-copr-user:
+      user: copr
+      rsync_args: --relative /home/copr/provision
+      command: rsnapshot_copr_backend

playbooks/groups/copr-backend.yml

@@ -62,3 +62,5 @@
   - copr/backend
   - role: messaging/base
     when: copr_messaging
+  - role: rsnapshot-push
+    when: env == "production"

roles/rsnapshot-push/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+rsnapshot_push_defaults:
+  timing_plans:
+    normal:
+      push: [0, 25]
+      daily: [86400, 10]
+      weekly: [604800, 4]
+      monthly: [2592000, 6]
+      yearly: [31536000, 3]
+
+    # we can't keep monthly increments for too large deltas
+    copr_be:
+      # Sunday / Wednesday
+      push: [0, 3]
+      biweekly: [1209600, 3]

roles/rsnapshot-push/tasks/main.yml

@@ -0,0 +1,55 @@
+---
+- name: backup script
+  template:
+    src: client-backup-script.sh.j2
+    dest: /usr/local/bin/"{{ item.value.command }}"
+    owner: "{{ item.value.user }}"
+    group: "{{ item.value.user }}"
+    mode: 0700
+  with_dict:
+  - "{{ rsnapshot_push.cases }}"
+  tags: rsnapshot_push
+
+- name: server-side case-specific backup dir
+  file:
+    path: "{{ '/'.join([rsnapshot_push.backup_dir, item.key]) }}"
+    state: directory
+    owner: "{{ item.value.user }}"
+    group: "{{ item.value.user }}"
+    mode: 0700
+  delegate_to: "{{ rsnapshot_push.server_host }}"
+  tags: rsnapshot_push
+
+- name: server-side custom rsnapshot daemon script
+  template:
+    src: server-daemon.sh.j2
+    dest: "{{ '/'.join([rsnapshot_push.backup_dir, item.key, 'sync-daemon']) }}"
+    owner: "{{ item.value.user }}"
+    group: "{{ item.value.user }}"
+    mode: 0700
+  with_dict:
+  - "{{ rsnapshot_push.cases }}"
+  delegate_to: "{{ rsnapshot_push.server_host }}"
+  tags: rsnapshot_push
+
+- name: rsnapshot call wrapper
+  template:
+    src: server-rsnapshot.py.j2
+    dest: "{{ '/'.join([rsnapshot_push.backup_dir, item.key, 'rsnapshot']) }}"
+    owner: "{{ item.value.user }}"
+    group: "{{ item.value.user }}"
+    mode: 0700
+  with_dict:
+  - "{{ rsnapshot_push.cases }}"
+  delegate_to: "{{ rsnapshot_push.server_host }}"
+  tags: rsnapshot_push
+
+- name: backup praiskup data
+  cron: name="backup documents - {{ item.key }}"
+        minute="0/5"
+        hour="*"
+        user={{ item.value.user }}
+        job=/usr/local/bin/"{{ item.value.command }}"
+  with_dict:
+  - "{{ rsnapshot_push.cases }}"
+  tags: rsnapshot_push

roles/rsnapshot-push/templates/client-backup-script.sh.j2

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+# the ::push is defined server-side target
+
+exec rsync -av --xattrs --acls \
+    --delete --delete-excluded \
+    {{ item.value.rsync_args }} \
+    {{ item.value.user }}@{{ rsnapshot_push.server_host }}::push

roles/rsnapshot-push/templates/server-daemon.sh.j2

@@ -0,0 +1,57 @@
+#! /bin/bash
+
+dirname=$(dirname "$(readlink -f "$0")" )
+rsync_config=$dirname/rsync.conf
+rsnapshot_config=$dirname/rsnapshot.conf
+
+backup=$dirname/backup
+sync_to=$backup/.sync
+lock="$dirname/.rsync.lock"
+mkdir -p "$sync_to"
+
+cat >"$rsnapshot_config" <<EOF
+config_version	1.2
+
+cmd_cp		/usr/bin/cp
+cmd_rm		/usr/bin/rm
+cmd_rsync	/usr/bin/rsync
+cmd_du		/usr/bin/du
+cmd_ssh		/usr/bin/ssh
+
+{% if 'timing_plan' in item.value %}
+{% set timing_plan = item.value.timing_plan %}
+{% else %}
+{% set timing_plan = 'normal' %}
+{% endif %}
+{% if timing_plan in rsnapshot_push_defaults.timing_plans %}
+{% set plan = rsnapshot_push_defaults.timing_plans[timing_plan] %}
+{% endif %}
+{% if 'timing_plans' in rsnapshot_push and timing_plan in rsnapshot_push.timing_plans %}
+{% set plan = rsnapshot_push.timing_plans[timing_plan] %}
+{% endif %}
+
+{% for name, spec in plan.items() %}
+retain		{{ name }}	{{ spec[1] }}
+{% endfor %}
+
+sync_first	1
+
+snapshot_root	$backup
+backup		$sync_to	./
+logfile		$dirname/rsnapshot.log
+lockfile	$dirname/rsnapshot.pid
+EOF
+
+cat >"$rsync_config" <<EOF
+[push]
+path = $sync_to
+use chroot = 0
+read only = 0
+write only = 1
+fake super = 1
+max connections = 1
+lock file = $lock
+post-xfer exec = $dirname/rsnapshot
+EOF
+
+/usr/bin/rsync --server --daemon "--config=$rsync_config" .

roles/rsnapshot-push/templates/server-rsnapshot.py.j2

@@ -0,0 +1,71 @@
+#! /usr/bin/python3
+
+"""
+Rotate backups with appropriate rsnapshot level(s).
+{% if 'timing_plan' in item.value %}
+{% set timing_plan = item.value.timing_plan %}
+{% else %}
+{% set timing_plan = 'normal' %}
+{% endif %}
+{% if timing_plan in rsnapshot_push_defaults.timing_plans %}
+{% set plan = rsnapshot_push_defaults.timing_plans[timing_plan] %}
+{% endif %}
+{% if 'timing_plans' in rsnapshot_push and timing_plan in rsnapshot_push.timing_plans %}
+{% set plan = rsnapshot_push.timing_plans[timing_plan] %}
+{% endif %}
+"""
+
+import json
+import time
+import subprocess
+
+DB = "{{ '/'.join([rsnapshot_push.backup_dir, item.key, 'rsnapshot_push.db']) }}"
+CONFIG = "{{ '/'.join([rsnapshot_push.backup_dir, item.key, 'rsnapshot.conf']) }}"
+LEVELS = {
+{% for name, spec in plan.items() %}
+    '{{ name }}': {{ spec[0] }},
+{% endfor %}
+}
+
+
+def _get_db():
+    try:
+        with open(DB, 'r') as fdb:
+            db_dict = json.loads(fdb.read())
+    except FileNotFoundError:
+        db_dict = {}
+    # initiate the levels which are not yet in DB
+    now = time.time()
+    force_reset = False
+    for key in LEVELS:
+        if key not in db_dict or force_reset:
+            db_dict[key] = now
+            force_reset = True
+    return db_dict
+
+
+def rotate(database):
+    """ rotate backups as needed, per last runs stored in database """
+    now = time.time()
+    for level, delay in sorted(LEVELS.items(), key=lambda x: x[1]):
+        cmd = ['/bin/rsnapshot', '-c', CONFIG, level]
+        last_run = database[level]
+        if (now - last_run) > delay:
+            print("running " + ' '.join(cmd))
+            subprocess.check_call(cmd)
+            database[level] = now
+        else:
+            print("skipping " + level)
+
+
+def _main():
+    database = _get_db()
+    try:
+        rotate(database)
+    finally:
+        with open(DB, 'w') as fdb:
+            fdb.write(json.dumps(database))
+
+
+if __name__ == "__main__":
+    _main()