diff --git a/files/scripts/create-filelist b/files/scripts/create-filelist index adf8104c61..eeba9d0898 100755 --- a/files/scripts/create-filelist +++ b/files/scripts/create-filelist @@ -63,7 +63,7 @@ def parseopts(): p.add_argument('-C', '--checksum-file', action='append', dest='checksum_files', help='Include checksums of all instances of the specified file.') p.add_argument('-s', '--skip', action='store_true', - help='Skip the --filelist file in the top directory') + help='Skip the file lists in the top directory') p.add_argument('-S', '--skip-file', action='append', dest='skip_files', help='Skip the specified file in the top directory.') @@ -87,6 +87,8 @@ def parseopts(): if opts.skip: if not opts.timelist.name == '': opts.skip_files += [opts.timelist.name] + if not opts.filelist.name == '': + opts.skip_files += [opts.filelist.name] return opts @@ -114,14 +116,15 @@ def main(): # opts.timelist.write('{0}\t{1}\t{2}\n'.format(modtime, ftype, entry.path[2:])) print('{0}\t{1}\t{2}\t{3}'.format(modtime, ftype, size, entry.path[2:]), file=opts.timelist) - if not checksums: - sys.exit(0) - print('\n[Checksums SHA1]', file=opts.timelist) + # It's OK if the checksum section is empty, but we should include it anyway + # as the client expects it. for f in sorted(checksums): print('{0}\t{1}'.format(sha1(f), f), file=opts.timelist) + print('\n[End]', file=opts.timelist) + if __name__ == '__main__': main() diff --git a/files/scripts/update-fullfiletimelist b/files/scripts/update-fullfiletimelist index 44de5f2121..016ca8ed6b 100755 --- a/files/scripts/update-fullfiletimelist +++ b/files/scripts/update-fullfiletimelist @@ -1,29 +1,100 @@ #!/bin/bash + +# Note: this is only an example of how you'd call create-filelist. Edit to fit +# your requirements. Note that you must supply a valid path for the lockfile, +# and it must be outside of your repository unless you want that lockfile to +# show up in your file lists. + # Takes a list of module names. Generates file lists for all of them and them # moves them into place at once. If you are creating hardlinks between rsync # modules, it is required that you update the file lists of both mirrors at the # same time. Otherwise the clients may make separate copies of the files. -mods=$* - -if [[ -z $mods ]]; then - echo "usage: $0 [ ...]" - exit 2 -fi - -CREATE=/usr/local/bin/create-filelist - -# A single lockfile for everything we're modifying -LOCKFILE=/srv/.lock.create-filelist - # The directory where all of the modules live -TOPD=/srv/pub +# Or pass it with -t +TOPD=/srv/mirror/pub + +# The modules to process. Or pass them on the command line. +MODS=() + +# Path to the create-filelist program. +# Or specify it with -p. +CREATE=/usr/local/bin/create-filelist # These strings will be eval'ed later with $mod replaced by its value in # context. FILELIST=fullfilelist TIMELIST='fullfiletimelist-$mod' +usage () { + echo + echo "Usage: $0 [-l lockfile] [-p creator path] [-t top directory] module [module ...]" + echo + echo " -l: Path to the lock file" + echo " -p: Path to the create-filelist program" + echo " -t: Path to directory containing modules" + echo + echo "At least one module to process must be provided." + echo "All paths must be absolute." +} + +while [[ $# > 0 ]]; do + opt=$1 + case $opt in + -l) + LOCKFILE=$(realpath $2) + shift + ;; + -p) + CREATE=$(realpath $2) + shift + ;; + -t) + TOPD=$(realpath $2) + shift + ;; + -*) + (>&2 echo "Unknown option $opt."; usage) + exit 1 + ;; + *) # Remaining args are modules + MODS+=($opt) + ;; + esac + shift +done + +if [[ -z $LOCKFILE ]]; then + (>&2 echo "Must specify LOCKFILE, either by editing the source or via the -l option."; usage) + exit 2 +fi +if [[ ! -d $(dirname $LOCKFILE) ]]; then + (>&2 echo "Given directory $(dirname $LOCKFILE) does not exist."; usage) + exit 2 +fi +if [[ ! -f $CREATE ]]; then + (>&2 echo "Specified executable $CREATE does not exist."; usage) + exit 2 +fi + +if [[ ! -d $TOPD ]]; then + (>&2 echo "Provided directory $TOPD does not exist."; usage) + exit 2 +fi + +if [[ ${#MODS[@]} -eq 0 ]]; then + (>&2 echo "No modules specified"; usage) + exit 2 +fi + +tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX) +if [[ $? -ne 0 ]]; then + (>&2 echo "Creating temporary directory failed?") + exit 1 +fi +trap "rm -rf $tmpd" EXIT +cd $tmpd + ( # We want to wait forever until we can do what we're asked flock -x 9 @@ -33,42 +104,46 @@ TIMELIST='fullfiletimelist-$mod' # flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up # Don't change the '9', unless you change the last line of this script. - tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX) - trap "rm -rf $tmpd" EXIT - cd $tmpd - - # Create all of the filelists - for mod in $mods; do - $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod - done - - # Now diff the file lists and delete the ones which didn't change - for mod in $mods; do + for mod in $MODS; do currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} + flname=$(basename $currentfl) + tlname=$(basename $currenttl) + + $CREATE -c -s -d $TOPD/$mod -f $flname -t $tlname # If a file list exsts and doesn't differ from what we just generated, # delete the latter. - if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then - rm -f $fl + if [[ -f $currentfl ]] && diff -q $currentfl $flname > /dev/null; then + rm -f $flname fi - if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then - rm -f $fl + if [[ -f $currenttl ]] && diff -q $currenttl $tlname > /dev/null; then + rm -f $tlname fi done - # And finally, move all of the files which need updating into place - for mod in $mods; do + # Now we have the new file lists but in a temporary directory which + # probably isn't on the same filesystem. Copy them to temporary files in + # the right place. + for mod in $MODS; do currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} + flname=$(basename $currentfl) + fldir=$(dirname $currentfl) + tlname=$(basename $currenttl) + tldir=$(dirname $currenttl) - if [[ -f fl-$mod ]]; then - chmod 644 fl-$mod - cp -p fl-$mod $currentfl + if [[ -f $flname ]]; then + tmpf=$(mktemp -p $fldir $flname.XXXXXXXXXX) + cp -p $flname $tmpf + chmod 644 $tmpf + mv $tmpf $currentfl fi - if [[ -f tl-$mod ]]; then - chmod 644 tl-$mod - cp -p tl-$mod $currenttl + if [[ -f $tlname ]]; then + tmpf=$(mktemp -p $tldir $tlname.XXXXXXXXXX) + cp -p $tlname $tmpf + chmod 644 $tmpf + mv $tmpf $currenttl fi done diff --git a/playbooks/groups/secondary.yml b/playbooks/groups/secondary.yml index be21c7eab7..f86539d4a3 100644 --- a/playbooks/groups/secondary.yml +++ b/playbooks/groups/secondary.yml @@ -67,7 +67,7 @@ - name: Update fullfiletimelist job cron: name="update-fullfiletimelist" hour="*" minute="55" user="root" - job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist alt'" + job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /srv/pub alt'" cron_file=update-fullfiletimelist handlers: diff --git a/roles/bodhi2/backend/files/update-fullfilelist b/roles/bodhi2/backend/files/update-fullfilelist deleted file mode 100755 index 65e4ef4924..0000000000 --- a/roles/bodhi2/backend/files/update-fullfilelist +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# Takes a list of module names. Generates file lists for all of them and them -# moves them into place at once. If you are creating hardlinks between rsync -# modules, it is required that you update the file lists of both mirrors at the -# same time. Otherwise the clients may make separate copies of the files. - -mods=$* - -if [[ -z $mods ]]; then - echo "usage: $0 [ ...]" - exit 2 -fi - -CREATE=/usr/local/bin/create-filelist - -# A single lockfile for everything we're modifying -LOCKFILE=.lock.create-filelist - -# The directory where all of the modules live -TOPD=/pub - -# These strings will be eval'ed later with $mod replaced by its value in -# context. -FILELIST=fullfilelist -TIMELIST='fullfiletimelist-$mod' - -( - # We want to wait forever until we can do what we're asked - flock -x 9 - - # If you don't want to wait forever, try one of the following: - # flock -n 9 || exit 1 - Gives up immediately - # flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up - # Don't change the '9', unless you change the last line of this script. - - tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX) - trap "rm -rf $tmpd" EXIT - cd $tmpd - - # Create all of the filelists - for mod in $mods; do - $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod - done - - # Now diff the file lists and delete the ones which didn't change - for mod in $mods; do - currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} - currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} - - # If a file list exsts and doesn't differ from what we just generated, - # delete the latter. - if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then - rm -f $fl - fi - if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then - rm -f $fl - fi - done - - # And finally, move all of the files which need updating into place - for mod in $mods; do - currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} - currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} - - if [[ -f fl-$mod ]]; then - chmod 644 fl-$mod - cp -p fl-$mod $currentfl - fi - if [[ -f tl-$mod ]]; then - chmod 644 tl-$mod - cp -p tl-$mod $currenttl - fi - done - -) 9>$LOCKFILE diff --git a/roles/bodhi2/backend/tasks/main.yml b/roles/bodhi2/backend/tasks/main.yml index 4a6b640ad2..4d39c684b1 100644 --- a/roles/bodhi2/backend/tasks/main.yml +++ b/roles/bodhi2/backend/tasks/main.yml @@ -211,8 +211,8 @@ - bodhi - cron -- name: put update-fullfilelist in place - copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist mode=0755 +- name: put update-fullfiletimelist in place + copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755 when: inventory_hostname.startswith('bodhi-backend01') and env == "production" tags: - config @@ -229,7 +229,7 @@ - name: Updates sync cron job. cron: name="updates-sync" minute="15,45" user="ftpsync" - job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfilelist fedora'" + job="/usr/local/bin/lock-wrapper fedora-updates-push '/usr/local/bin/fedora-updates-push && /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora'" cron_file=updates-sync when: inventory_hostname.startswith('bodhi-backend01') and env == "production" tags: diff --git a/roles/releng/files/branched b/roles/releng/files/branched index 4430d672f1..884e94ce37 100644 --- a/roles/releng/files/branched +++ b/roles/releng/files/branched @@ -1,3 +1,3 @@ # branched compose MAILTO=releng-cron@lists.fedoraproject.org -#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f24 && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfilelist fedora +#15 7 * * * root TMPDIR=`mktemp -d /tmp/branched.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && git checkout f24 && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora diff --git a/roles/releng/files/rawhide b/roles/releng/files/rawhide index d85c9ce30f..16070eced2 100644 --- a/roles/releng/files/rawhide +++ b/roles/releng/files/rawhide @@ -1,3 +1,3 @@ # rawhide compose MAILTO=releng-cron@lists.fedoraproject.org -15 5 * * * root TMPDIR=`mktemp -d /tmp/rawhide.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfilelist fedora +15 5 * * * root TMPDIR=`mktemp -d /tmp/rawhide.XXXXXX` && cd $TMPDIR && git clone https://pagure.io/pungi-fedora.git && cd pungi-fedora && LANG=en_US.UTF-8 ./nightly.sh && sudo -u ftpsync /usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /pub fedora diff --git a/roles/releng/files/update-fullfilelist b/roles/releng/files/update-fullfilelist deleted file mode 100755 index 65e4ef4924..0000000000 --- a/roles/releng/files/update-fullfilelist +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# Takes a list of module names. Generates file lists for all of them and them -# moves them into place at once. If you are creating hardlinks between rsync -# modules, it is required that you update the file lists of both mirrors at the -# same time. Otherwise the clients may make separate copies of the files. - -mods=$* - -if [[ -z $mods ]]; then - echo "usage: $0 [ ...]" - exit 2 -fi - -CREATE=/usr/local/bin/create-filelist - -# A single lockfile for everything we're modifying -LOCKFILE=.lock.create-filelist - -# The directory where all of the modules live -TOPD=/pub - -# These strings will be eval'ed later with $mod replaced by its value in -# context. -FILELIST=fullfilelist -TIMELIST='fullfiletimelist-$mod' - -( - # We want to wait forever until we can do what we're asked - flock -x 9 - - # If you don't want to wait forever, try one of the following: - # flock -n 9 || exit 1 - Gives up immediately - # flock -w 120 9 || exit 1 - Waits 120 seconds and then gives up - # Don't change the '9', unless you change the last line of this script. - - tmpd=$(mktemp -d -t create-filelist.XXXXXXXXXX) - trap "rm -rf $tmpd" EXIT - cd $tmpd - - # Create all of the filelists - for mod in $mods; do - $CREATE -c -s -d $TOPD/$mod -f fl-$mod -t tl-$mod - done - - # Now diff the file lists and delete the ones which didn't change - for mod in $mods; do - currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} - currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} - - # If a file list exsts and doesn't differ from what we just generated, - # delete the latter. - if [[ -f $currentfl ]] && diff -q fl-$mod $currentfl > /dev/null; then - rm -f $fl - fi - if [[ -f $currenttl ]] && diff -q tl-$mod $currenttl > /dev/null; then - rm -f $fl - fi - done - - # And finally, move all of the files which need updating into place - for mod in $mods; do - currentfl=$TOPD/$mod/${FILELIST/'$mod'/$mod} - currenttl=$TOPD/$mod/${TIMELIST/'$mod'/$mod} - - if [[ -f fl-$mod ]]; then - chmod 644 fl-$mod - cp -p fl-$mod $currentfl - fi - if [[ -f tl-$mod ]]; then - chmod 644 tl-$mod - cp -p tl-$mod $currenttl - fi - done - -) 9>$LOCKFILE diff --git a/roles/releng/tasks/main.yml b/roles/releng/tasks/main.yml index 448c1bdcaf..0745ccc7f5 100644 --- a/roles/releng/tasks/main.yml +++ b/roles/releng/tasks/main.yml @@ -21,8 +21,8 @@ - name: add create-filelist script from quick-fedora-mirror copy: src="{{ files }}/scripts/create-filelist" dest=/usr/local/bin/create-filelist mode=0755 owner=ftpsync group=ftpsync -- name: add the ftpsync update-fullfilelist script - copy: src=update-fullfilelist dest=/usr/local/bin/update-fullfilelist owner=ftpsync group=ftpsync mode=555 +- name: add the ftpsync update-fullfiletimelist script + copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755 - name: add masher group group: name=masher gid=751 system=yes state=present