From b9a6da7eeb9ee14ee7f9858ccba4daad5e0f00a1 Mon Sep 17 00:00:00 2001 From: Mikolaj Izdebski Date: Wed, 28 Feb 2018 18:12:08 +0000 Subject: [PATCH] Try some pipe buffering for faster db-koji stg sync --- playbooks/manual/staging-sync/koji.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/playbooks/manual/staging-sync/koji.yml b/playbooks/manual/staging-sync/koji.yml index a884b9e68a..d9326a7bcc 100644 --- a/playbooks/manual/staging-sync/koji.yml +++ b/playbooks/manual/staging-sync/koji.yml @@ -66,9 +66,16 @@ - postgresql_db: name=koji state=absent - postgresql_db: name=koji owner=koji # buildroot_listing is excluded from the sync to save some time -# TODO: add pipe buffering +# TODO: install mbuffer on db-koji.stg - name: Import the prod db. This will take quite a while. Go get a snack! - shell: curl https://infrastructure.fedoraproject.org/infra/db-dumps/koji.dump.xz | xzcat | sed '/COPY buildroot_listing /,/\./d' | psql koji + shell: > + curl -s -S https://infrastructure.fedoraproject.org/infra/db-dumps/koji.dump.xz | + mbuffer -q -m 10M -l /tmp/mbuffer-curl-to-xz.log | + xzcat | + mbuffer -q -m 100M -l /tmp/mbuffer-xz-to-sed.log | + sed '/COPY buildroot_listing /,/\./d' | + mbuffer -q -m 1G -l /tmp/mbuffer-sed-to-psql.log | + psql koji - name: repoint all the prod rpm entries at the secondary volume (and other stuff) shell: psql -1 koji