Add grokmirror mirror role for backup server.

Cron jobs are commented out for now until we have an initial mirror.

inventory/host_vars/backup01.phx2.fedoraproject.org

@@ -7,3 +7,5 @@ datacenter: phx2
 eth0_ip: 10.5.126.161
 eth1_ip: 10.5.127.24
 gw: 10.5.126.254
+
+grokmirror_topdir: /fedora_backups/grokmirror

playbooks/groups/backup-server.yml

@@ -26,6 +26,7 @@
       nfs_mount_opts: "rw,hard,bg,intr,noatime,nodev,nosuid,sec=sys,nfsvers=3",
       nfs_src_dir: 'fedora_backups' }
   - openvpn/client
+  - grokmirror-mirror
 
   tasks:
   - include_tasks: "{{ tasks_path }}/yumrepos.yml"

roles/grokmirror-mirror/tasks/main.yml

@@ -0,0 +1,34 @@
+---
+#
+# Setup a host to mirror our various git repos with grokmirror
+#
+
+- name: install grokmirror
+  package: name=python-grokmirror state=installed
+  tags:
+  - grokmirror-mirror
+
+- name: create directory to mirror repos to
+  file: dest={{grokmirror_topdir}} mode=0755 state=directory
+  tags:
+  - grokmirror-mirror
+
+- name: install grokmirror repos config file from template
+  template: src=repos.conf dest={{grokmirror_topdir)/repos.conf user=root group=root mode=644
+  tags:
+  - grokmirror-mirror
+
+- name: install grokmirror fsck config file from template
+  template: src=fsck.conf dest={{grokmirror_topdir)/fsck.conf user=root group=root mode=644
+  tags:
+  - grokmirror-mirror
+
+- name: install grokmirror repos cron job
+  template: src=grokmirror.cron dest=/etc/cron.d/grokmirror.cron user=root group=root mode=644
+  tags:
+  - grokmirror-mirror
+
+- name: install grokmirror fsck cron job
+  template: src=grokfsck.cron dest=/etc/cron.d/grokfsck.cron user=root group=root mode=644
+  tags:
+  - grokmirror-mirror

roles/grokmirror-mirror/templates/fsck.conf

@@ -0,0 +1,66 @@
+# You can have multiple sections, just name them appropriately
+[src.fedoraproject.org]
+# Where is the manifest containing the list of repositories?
+manifest = {{ grokmirror_topdir }}/src.fedoraproject.org/manifest.js.gz
+#
+# Where are the repositories kept?
+#toplevel = /var/lib/git/mirror
+toplevel = {{ grokmirror_topdir }}/src.fedoraproject.org
+#
+# Where do we put the logs?
+#log = /var/log/mirror/kernelorg-fsck.log
+log = /var/log/grokmirror/src.fedoraproject.org-fsck.log
+#
+# Log level can be "info" or "debug"
+#loglevel = info
+loglevel = info
+#
+# Make sure there is only one instance of grok-fsck running by
+# trying to exclusive-lock this file before we do anything.
+lock = /var/tmp/kernelorg-fsck.lock
+#
+# Where to keep the status file
+#statusfile = /var/lib/mirror/kernelorg-fsck.js
+statusfile = /var/tmp/kernelorg-fsck.js
+#
+# How often should we check each repository, in days.
+# Any newly added repository will have the first check within a random
+# period of 0 and $frequency, and then every $frequency after that,
+# to assure that not all repositories are checked on the same day.
+# Don't set to less than 7 unless you only mirror a few repositories
+# (or really like to thrash your disks).
+#frequency = 30
+frequency = 30
+#
+# Some errors are relatively benign and can be safely ignored. Add matching
+# substrings to this field to ignore them.
+ignore_errors = dangling commit
+                dangling blob
+                notice: HEAD points to an unborn branch
+                notice: No default references
+                contains zero-padded file modes
+#
+# Repack the repositories after calling git-fsck.
+# This will especially save you space if you have shared repositories.
+# To check if you have shared repositories, look at your manifest.js.gz to
+# see if any repository definition has a "reference" key.
+repack = yes
+#
+# Default repack flags are -A -d -l -q, but you can specify your own here
+repack_flags = -A -d -l -q
+#
+# Once in a while you should repack your repositories more thoroughly,
+# by passing a -f flag and using a larger window/depth. You shouldn't need
+# to do this more frequently than once every 10-15 regular repacks.
+#
+# Trigger a full repack every N times.
+full_repack_every = 10
+#
+# What flags should we use when doing a full repack.
+full_repack_flags = -A -d -l -q -f --window=200 --depth=50
+#
+# Run git-prune to remove obsolete old objects if no other repositories are
+# using the repo in their objects/info/alternates. If other repositories
+# are relying on this repo via alternates, it will not be pruned to avoid
+# potential corruption.
+prune = yes

roles/grokmirror-mirror/templates/grokfsck.cron

@@ -0,0 +1 @@
+#00 02 * * * root /usr/bin/grok-fsck -c {{ grokmirror_topdir }}/fsck.conf

roles/grokmirror-mirror/templates/grokmirror.cron

@@ -0,0 +1 @@
+#* * * * * root /usr/bin/grok-pull -p -c {{ grokmirror_topdir }}/repos.conf

roles/grokmirror-mirror/templates/repos.conf

@@ -0,0 +1,106 @@
+# You can pull from multiple grok mirrors, just create
+# a separate section for each mirror. The name can be anything.
+[src.fedoraproject.org]
+# The host part of the mirror you're pulling from.
+#site = git://git.kernel.org
+site = https://src.fedoraproject.org
+#
+# Where the grok manifest is published. The following protocols
+# are supported at this time:
+# http:// or https:// using If-Modified-Since http header
+# file:// (when manifest file is on NFS, for example)
+#manifest = http://git.kernel.org/manifest.js.gz
+manifest = https://src.fedoraproject.org/grokmirror/manifest.js.gz
+#
+# Where are we going to put the mirror on our disk?
+#toplevel = /var/lib/git/mirror
+toplevel = {{ grokmirror_topdir }}/src.fedoraproject.org/
+#
+# Where do we store our own manifest? Usually in the toplevel.
+#mymanifest = /var/lib/git/mirror/manifest.js.gz
+mymanifest = {{ grokmirror_topdir }}/src.fedoraproject.org/manifest.js.gz
+#
+# Write out projects.list that can be used by gitweb or cgit.
+# Leave blank if you don't want a projects.list.
+#projectslist = /var/lib/git/mirror/projects.list
+projectslist =
+#
+# When generating projects.list, start at this subpath instead
+# of at the toplevel. Useful when mirroring kernel or when generating
+# multiple gitweb/cgit configurations for the same tree.
+#projectslist_trimtop = /pub/scm/
+projectslist_trimtop = /pub/scm/
+#
+# The default behaviour is to use cross-repository references (if present)
+# to set git alternates between projects. This helps both significantly
+# reduce the bandwidth during cloning and dramatically reduce the mirror size
+# on disk. If for some reason you always want to create non-shared repositories,
+# set ignore_repo_references to "yes"
+# NOTE: this will NOT remove alternates for repositories already mirrored. You
+# will need to run "git repack -a" and then manually remove the
+# objects/info/alternates file for each repository.
+#ignore_repo_references = no
+#
+# When generating projects.list, also create entries for symlinks.
+# Otherwise we assume they are just legacy and keep them out of
+# web interfaces.
+#projectslist_symlinks = yes
+projectslist_symlinks = no
+#
+# A simple hook to execute whenever a repository is modified.
+# It passes the full path to the git repository modified as the only
+# argument.
+#post_update_hook = /usr/local/bin/make-git-fairies-appear
+post_update_hook =
+#
+# This prevents catastrophic mirror purges when our upstream master gives us
+# a manifest that is dramatically smaller than ours. The default is to refuse
+# the purge if the remote manifest has over 10% fewer repositories than what we
+# have, or in other words, if we have 100 repos and the remote manifest has
+# shrunk to 95 repos or less, we refuse to purge, suspecting that something has
+# gone wrong. You can set purgeprotect to a higher percentage, or override
+# it entirely with --force-purge commandline flag.
+#purgeprotect = 5
+purgeprotect = 5
+#
+# If owner is not specified in the manifest, who should be listed
+# as the default owner in tools like gitweb or cgit?
+#default_owner = Grokmirror User
+default_owner = Grokmirror User
+#
+# Where do we put the logs?
+#log = /var/log/mirror/kernelorg.log
+log = /var/log/grokmirror/src.fedoraproject.org.log
+#
+# Log level can be "info" or "debug"
+#loglevel = info
+loglevel = info
+#
+# To speed up updates, grok-pull will use multiple threads. Please be
+# considerate to the mirror you're pulling from and don't set this very
+# high. You may also run into per-ip multiple session limits, so leave this
+# number at a nice low setting.
+#pull_threads = 5
+pull_threads = 5
+#
+# Use shell-globbing to list the repositories you would like to mirror.
+# If you want to mirror everything, just say "*". Separate multiple entries
+# with newline plus tab. Examples:
+#
+# mirror everything:
+#include = *
+#
+# mirror just the main kernel sources:
+#include = /pub/scm/linux/kernel/git/torvalds/linux.git
+#          /pub/scm/linux/kernel/git/stable/linux-stable.git
+#          /pub/scm/linux/kernel/git/next/linux-next.git
+#
+# mirror just git:
+#include = /pub/scm/git/*
+include = *
+#
+# This is processed after the include. If you want to exclude some specific
+# entries from an all-inclusive globbing above. E.g., to exclude all linux-2.4
+# git sources:
+#exclude = */linux-2.4*
+exclude =