diff --git a/roles/grokmirror_mirror/tasks/main.yml b/roles/grokmirror_mirror/tasks/main.yml new file mode 100644 index 0000000000..22063397a5 --- /dev/null +++ b/roles/grokmirror_mirror/tasks/main.yml @@ -0,0 +1,34 @@ +--- +# +# Setup a host to mirror our various git repos with grokmirror +# + +- name: install grokmirror + package: name=python-grokmirror state=installed + tags: + - grokmirror-mirror + +- name: create directory to mirror repos to + file: dest={{grokmirror_topdir}} mode=0755 state=directory + tags: + - grokmirror-mirror + +- name: install grokmirror repos config file from template + template: src=repos.conf dest={{grokmirror_topdir)/repos.conf user=root group=root mode=644 + tags: + - grokmirror-mirror + +- name: install grokmirror fsck config file from template + template: src=fsck.conf dest={{grokmirror_topdir)/fsck.conf user=root group=root mode=644 + tags: + - grokmirror-mirror + +- name: install grokmirror repos cron job + template: src=grokmirror.cron dest=/etc/cron.d/grokmirror.cron user=root group=root mode=644 + tags: + - grokmirror-mirror + +- name: install grokmirror fsck cron job + template: src=grokfsck.cron dest=/etc/cron.d/grokfsck.cron user=root group=root mode=644 + tags: + - grokmirror-mirror diff --git a/roles/grokmirror_mirror/templates/fsck.conf b/roles/grokmirror_mirror/templates/fsck.conf new file mode 100644 index 0000000000..cefb17eaec --- /dev/null +++ b/roles/grokmirror_mirror/templates/fsck.conf @@ -0,0 +1,66 @@ +# You can have multiple sections, just name them appropriately +[src.fedoraproject.org] +# Where is the manifest containing the list of repositories? +manifest = {{ grokmirror_topdir }}/src.fedoraproject.org/manifest.js.gz +# +# Where are the repositories kept? +#toplevel = /var/lib/git/mirror +toplevel = {{ grokmirror_topdir }}/src.fedoraproject.org +# +# Where do we put the logs? +#log = /var/log/mirror/kernelorg-fsck.log +log = /var/log/grokmirror/src.fedoraproject.org-fsck.log +# +# Log level can be "info" or "debug" +#loglevel = info +loglevel = info +# +# Make sure there is only one instance of grok-fsck running by +# trying to exclusive-lock this file before we do anything. +lock = /var/tmp/kernelorg-fsck.lock +# +# Where to keep the status file +#statusfile = /var/lib/mirror/kernelorg-fsck.js +statusfile = /var/tmp/kernelorg-fsck.js +# +# How often should we check each repository, in days. +# Any newly added repository will have the first check within a random +# period of 0 and $frequency, and then every $frequency after that, +# to assure that not all repositories are checked on the same day. +# Don't set to less than 7 unless you only mirror a few repositories +# (or really like to thrash your disks). +#frequency = 30 +frequency = 30 +# +# Some errors are relatively benign and can be safely ignored. Add matching +# substrings to this field to ignore them. +ignore_errors = dangling commit + dangling blob + notice: HEAD points to an unborn branch + notice: No default references + contains zero-padded file modes +# +# Repack the repositories after calling git-fsck. +# This will especially save you space if you have shared repositories. +# To check if you have shared repositories, look at your manifest.js.gz to +# see if any repository definition has a "reference" key. +repack = yes +# +# Default repack flags are -A -d -l -q, but you can specify your own here +repack_flags = -A -d -l -q +# +# Once in a while you should repack your repositories more thoroughly, +# by passing a -f flag and using a larger window/depth. You shouldn't need +# to do this more frequently than once every 10-15 regular repacks. +# +# Trigger a full repack every N times. +full_repack_every = 10 +# +# What flags should we use when doing a full repack. +full_repack_flags = -A -d -l -q -f --window=200 --depth=50 +# +# Run git-prune to remove obsolete old objects if no other repositories are +# using the repo in their objects/info/alternates. If other repositories +# are relying on this repo via alternates, it will not be pruned to avoid +# potential corruption. +prune = yes diff --git a/roles/grokmirror_mirror/templates/grokfsck.cron b/roles/grokmirror_mirror/templates/grokfsck.cron new file mode 100644 index 0000000000..de33b86f91 --- /dev/null +++ b/roles/grokmirror_mirror/templates/grokfsck.cron @@ -0,0 +1 @@ +#00 02 * * * root /usr/bin/grok-fsck -c {{ grokmirror_topdir }}/fsck.conf diff --git a/roles/grokmirror_mirror/templates/grokmirror.cron b/roles/grokmirror_mirror/templates/grokmirror.cron new file mode 100644 index 0000000000..ed0320dd0e --- /dev/null +++ b/roles/grokmirror_mirror/templates/grokmirror.cron @@ -0,0 +1 @@ +#* * * * * root /usr/bin/grok-pull -p -c {{ grokmirror_topdir }}/repos.conf diff --git a/roles/grokmirror_mirror/templates/repos.conf b/roles/grokmirror_mirror/templates/repos.conf new file mode 100644 index 0000000000..dfe72194fc --- /dev/null +++ b/roles/grokmirror_mirror/templates/repos.conf @@ -0,0 +1,106 @@ +# You can pull from multiple grok mirrors, just create +# a separate section for each mirror. The name can be anything. +[src.fedoraproject.org] +# The host part of the mirror you're pulling from. +#site = git://git.kernel.org +site = https://src.fedoraproject.org +# +# Where the grok manifest is published. The following protocols +# are supported at this time: +# http:// or https:// using If-Modified-Since http header +# file:// (when manifest file is on NFS, for example) +#manifest = http://git.kernel.org/manifest.js.gz +manifest = https://src.fedoraproject.org/grokmirror/manifest.js.gz +# +# Where are we going to put the mirror on our disk? +#toplevel = /var/lib/git/mirror +toplevel = {{ grokmirror_topdir }}/src.fedoraproject.org/ +# +# Where do we store our own manifest? Usually in the toplevel. +#mymanifest = /var/lib/git/mirror/manifest.js.gz +mymanifest = {{ grokmirror_topdir }}/src.fedoraproject.org/manifest.js.gz +# +# Write out projects.list that can be used by gitweb or cgit. +# Leave blank if you don't want a projects.list. +#projectslist = /var/lib/git/mirror/projects.list +projectslist = +# +# When generating projects.list, start at this subpath instead +# of at the toplevel. Useful when mirroring kernel or when generating +# multiple gitweb/cgit configurations for the same tree. +#projectslist_trimtop = /pub/scm/ +projectslist_trimtop = /pub/scm/ +# +# The default behaviour is to use cross-repository references (if present) +# to set git alternates between projects. This helps both significantly +# reduce the bandwidth during cloning and dramatically reduce the mirror size +# on disk. If for some reason you always want to create non-shared repositories, +# set ignore_repo_references to "yes" +# NOTE: this will NOT remove alternates for repositories already mirrored. You +# will need to run "git repack -a" and then manually remove the +# objects/info/alternates file for each repository. +#ignore_repo_references = no +# +# When generating projects.list, also create entries for symlinks. +# Otherwise we assume they are just legacy and keep them out of +# web interfaces. +#projectslist_symlinks = yes +projectslist_symlinks = no +# +# A simple hook to execute whenever a repository is modified. +# It passes the full path to the git repository modified as the only +# argument. +#post_update_hook = /usr/local/bin/make-git-fairies-appear +post_update_hook = +# +# This prevents catastrophic mirror purges when our upstream master gives us +# a manifest that is dramatically smaller than ours. The default is to refuse +# the purge if the remote manifest has over 10% fewer repositories than what we +# have, or in other words, if we have 100 repos and the remote manifest has +# shrunk to 95 repos or less, we refuse to purge, suspecting that something has +# gone wrong. You can set purgeprotect to a higher percentage, or override +# it entirely with --force-purge commandline flag. +#purgeprotect = 5 +purgeprotect = 5 +# +# If owner is not specified in the manifest, who should be listed +# as the default owner in tools like gitweb or cgit? +#default_owner = Grokmirror User +default_owner = Grokmirror User +# +# Where do we put the logs? +#log = /var/log/mirror/kernelorg.log +log = /var/log/grokmirror/src.fedoraproject.org.log +# +# Log level can be "info" or "debug" +#loglevel = info +loglevel = info +# +# To speed up updates, grok-pull will use multiple threads. Please be +# considerate to the mirror you're pulling from and don't set this very +# high. You may also run into per-ip multiple session limits, so leave this +# number at a nice low setting. +#pull_threads = 5 +pull_threads = 5 +# +# Use shell-globbing to list the repositories you would like to mirror. +# If you want to mirror everything, just say "*". Separate multiple entries +# with newline plus tab. Examples: +# +# mirror everything: +#include = * +# +# mirror just the main kernel sources: +#include = /pub/scm/linux/kernel/git/torvalds/linux.git +# /pub/scm/linux/kernel/git/stable/linux-stable.git +# /pub/scm/linux/kernel/git/next/linux-next.git +# +# mirror just git: +#include = /pub/scm/git/* +include = * +# +# This is processed after the include. If you want to exclude some specific +# entries from an all-inclusive globbing above. E.g., to exclude all linux-2.4 +# git sources: +#exclude = */linux-2.4* +exclude =