Setup rhel and centos sync scripts
Setup rhel8-split scripts and cron job Signed-off-by: Mohan Boddu <mboddu@bhujji.com>
This commit is contained in:
Mohan Boddu
smooge
parent
46b327807f
commit
8ba8429972
7 changed files with 351 additions and 0 deletions
19
roles/batcave/files/centos-8-sync
Normal file
19
roles/batcave/files/centos-8-sync
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
#!/bin/sh
|
||||
|
||||
RSYNC_DESTDIR=/mnt/fedora/app/fi-repo/centos/centos-8/
|
||||
RSYNC_MOD="centos-epel/centos/8.1.1911/"
|
||||
|
||||
REPO_FROMDIR=/mnt/fedora/app/fi-repo/centos/centos-8/
|
||||
|
||||
mkdir -p ${RSYNC_DESTDIR}
|
||||
|
||||
RSYNC='/usr/bin/rsync'
|
||||
RS_OPT="-avSHP --numeric-ids"
|
||||
#RS_DEADLY="--delete --delete-excluded --delete-delay --delay-updates"
|
||||
RS_DEADLY="--delete --delete-delay --delay-updates"
|
||||
CENT_EXCLUDES="--exclude=isos --exclude=kickstart --exclude=s390x"
|
||||
|
||||
SERVER=centos-master01.iad2.fedoraproject.org
|
||||
|
||||
${RSYNC} ${RS_OPT} ${RS_DEADLY} ${CENT_EXCLUDES} ${SERVER}::${RSYNC_MOD} ${RSYNC_DESTDIR}
|
||||
|
||||
47
roles/batcave/files/rhel6-sync
Normal file
47
roles/batcave/files/rhel6-sync
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#!/bin/sh
|
||||
|
||||
RHEL6SYNCDIR=/srv/web/repo/rhel/rhel6
|
||||
RHEL6CACHEDIR=/var/tmp/reposync-rhel6
|
||||
|
||||
cd $RHEL6SYNCDIR
|
||||
|
||||
|
||||
mkdir -p ${RHEL6CACHEDIR}
|
||||
for i in i686 x86_64 ppc64; do
|
||||
mkdir -p ${RHEL6CACHEDIR}/${i}
|
||||
done
|
||||
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-optional-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686
|
||||
# 2019-09-17 RHEL does not have these anymore for i386
|
||||
#reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-ha-i386 -p i386 -a i686
|
||||
#reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-lb-i386 -p i386 -a i686
|
||||
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-optional-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-ha-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-lb-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64
|
||||
|
||||
#
|
||||
# rhel6 didn't have lb or ha repos for ppc64
|
||||
#
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-ppc64 -p ppc64 -a ppc64 --metadata-path ${RHEL6CACHEDIR}/ppc64
|
||||
reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-optional-ppc64 -p ppc64 -a ppc64 --metadata-path ${RHEL6CACHEDIR}/ppc64
|
||||
|
||||
for i in `ls -d /srv/web/repo/rhel/rhel6/i386/*`
|
||||
do
|
||||
cd $i
|
||||
createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 .
|
||||
done
|
||||
|
||||
for i in `ls -d /srv/web/repo/rhel/rhel6/x86_64/*`
|
||||
do
|
||||
cd $i
|
||||
createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 .
|
||||
done
|
||||
|
||||
for i in `ls -d /srv/web/repo/rhel/rhel6/ppc64/*`
|
||||
do
|
||||
cd $i
|
||||
createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 .
|
||||
done
|
||||
52
roles/batcave/files/rhel7-sync
Normal file
52
roles/batcave/files/rhel7-sync
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/bin/sh
|
||||
|
||||
RHEL7SYNCDIR=/srv/web/repo/rhel/rhel7
|
||||
SYNCDIR=/var/cache/reposync/
|
||||
YUMDIR=/var/tmp/reposync-rhel7/
|
||||
|
||||
## Need to download the metadata and the modular data
|
||||
ROPTS='--download-metadata -m'
|
||||
|
||||
# s390 isnt working yet
|
||||
ARCHES="ppc64le s390x x86_64"
|
||||
#ARCHES="x86_64"
|
||||
|
||||
mkdir -vp ${SYNCDIR} ${RHEL7SYNCDIR} ${YUMDIR}
|
||||
for A in ${ARCHES}; do
|
||||
mkdir -vp ${YUMDIR}/${A}
|
||||
done
|
||||
|
||||
cd $RHEL7SYNCDIR
|
||||
|
||||
##
|
||||
## Sync out RHEL-7 for appropriate arches
|
||||
for A in ${ARCHES}; do
|
||||
rm -rf ${SYNCDIR}/${A}
|
||||
CONF=yum-batcave-rhel7-${A}.conf
|
||||
REPOS="rhel-7-for-${A}-server-rpms rhel-7-ha-for-${A}-server-rpms rhel-7-extras-for-${A}-server-rpms rhel-7-optional-for-${A}-server-rpms rhel-7-devtools-for-${A}-server-rpms rhel-7-rhscl-for-${A}-server-rpms"
|
||||
# REPOS="rhel-7-rhscl-for-${A}-server-rpms"
|
||||
for R in ${REPOS}; do
|
||||
rm -rf ${YUMDIR}/${R}
|
||||
# Test to see if we have this repository in our tree
|
||||
grep -q ${R} ${RHEL7SYNCDIR}/${CONF}
|
||||
if [[ $? -eq 0 ]]; then
|
||||
echo "Syncing ${R} for ${A}"
|
||||
reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -p ${A} --metadata-path ${YUMDIR}/${A}
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Something went wrong with reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -a noarch -p ${A} --metadata-path ${YUMDIR}/${A}"
|
||||
continue
|
||||
fi
|
||||
## Try to create repo
|
||||
pushd ${RHEL7SYNCDIR}/${A}/${R} &> /dev/null
|
||||
createrepo_c -q --update --workers 6 .
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Something went wrong with createrepo ${A} ${R}"
|
||||
continue
|
||||
fi
|
||||
popd &> /dev/null
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
#ostree pull --repo=/mnt/fedora/app/fi-repo/rhel/rhel7/atomic/rhel7 --mirror redhat
|
||||
## This section is for all the trees which are not easily scriptable
|
||||
1
roles/batcave/files/rhel8-split.cron
Normal file
1
roles/batcave/files/rhel8-split.cron
Normal file
|
|
@ -0,0 +1 @@
|
|||
03 09 * * * root /usr/local/bin/rhel8-split.sh
|
||||
99
roles/batcave/files/rhel8-split.sh
Normal file
99
roles/batcave/files/rhel8-split.sh
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
#!/bin/bash
|
||||
HOMEDIR=/mnt/fedora/app/fi-repo/rhel/rhel8
|
||||
BINDIR=/usr/local/bin
|
||||
|
||||
ARCHES="aarch64 ppc64le s390x x86_64"
|
||||
DATE=$(date -Ih | sed 's/+.*//')
|
||||
|
||||
DATEDIR=${HOMEDIR}/koji/${DATE}
|
||||
|
||||
if [ -d ${DATEDIR} ]; then
|
||||
echo "Directory already exists. Please remove or fix"
|
||||
exit
|
||||
else
|
||||
mkdir -p ${DATEDIR}
|
||||
fi
|
||||
|
||||
for ARCH in ${ARCHES}; do
|
||||
# The archdir is where we daily download updates for rhel8
|
||||
ARCHDIR=${HOMEDIR}/${ARCH}
|
||||
if [ ! -d ${ARCHDIR} ]; then
|
||||
echo "Unable to find ${ARCHDIR}"
|
||||
exit
|
||||
fi
|
||||
|
||||
# We consolidate all of the default repositories and remerge them
|
||||
# in a daily tree. This allows us to point koji at a particular
|
||||
# day if we have specific build concerns.
|
||||
OUTDIR=${DATEDIR}/${ARCH}
|
||||
mkdir -p ${OUTDIR}
|
||||
if [ ! -d ${OUTDIR} ]; then
|
||||
echo "Unable to find ${ARCHDIR}"
|
||||
exit
|
||||
else
|
||||
cd ${OUTDIR}
|
||||
fi
|
||||
|
||||
# Begin splitting the various packages into their subtrees
|
||||
${BINDIR}/splitter.py --action hardlink --target RHEL-8-001 ${ARCHDIR}/rhel-8-for-${ARCH}-baseos-rpms/ --only-defaults &> /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "splitter ${ARCH} baseos failed"
|
||||
exit
|
||||
fi
|
||||
${BINDIR}/splitter.py --action hardlink --target RHEL-8-002 ${ARCHDIR}/rhel-8-for-${ARCH}-appstream-rpms/ --only-defaults &> /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "splitter ${ARCH} appstream failed"
|
||||
exit
|
||||
fi
|
||||
${BINDIR}/splitter.py --action hardlink --target RHEL-8-003 ${ARCHDIR}/codeready-builder-for-rhel-8-${ARCH}-rpms/ &> /dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "splitter ${ARCH} codeready failed"
|
||||
exit
|
||||
fi
|
||||
|
||||
# Copy the various module trees into RHEL-8-001 where we want them
|
||||
# to work.
|
||||
echo "Moving data to ${ARCH}/RHEL-8-001"
|
||||
cp -anlr RHEL-8-002/* RHEL-8-001
|
||||
cp -anlr RHEL-8-003/* RHEL-8-001
|
||||
# Go into the main tree
|
||||
pushd RHEL-8-001
|
||||
|
||||
# Mergerepo didn't work so lets just createrepo in the top directory.
|
||||
createrepo_c . &> /dev/null
|
||||
popd
|
||||
|
||||
# Cleanup the trash
|
||||
rm -rf RHEL-8-002 RHEL-8-003
|
||||
#loop to the next
|
||||
done
|
||||
|
||||
## Set up the builds so they are pointing to the last working version
|
||||
cd ${HOMEDIR}/koji/
|
||||
if [[ -e staged ]]; then
|
||||
if [[ -h staged ]]; then
|
||||
rm -f staged
|
||||
else
|
||||
echo "Unable to remove staged. it is not a symbolic link"
|
||||
exit
|
||||
fi
|
||||
else
|
||||
echo "No staged link found"
|
||||
fi
|
||||
|
||||
echo "Linking ${DATE} to staged"
|
||||
ln -s ${DATE} staged
|
||||
|
||||
|
||||
for ARCH in ${ARCHES}; do
|
||||
pushd latest/
|
||||
mkdir -p ${ARCH}
|
||||
dnf --disablerepo=\* --enablerepo=RHEL-8-001 --repofrompath=RHEL-8-001,https://infrastructure.fedoraproject.org/repo/rhel/rhel8/koji/staged/${ARCH}/RHEL-8-001/ reposync -a ${ARCH} -a noarch -p ${ARCH} --newest --delete &> /dev/null
|
||||
if [[ $? -eq 0 ]]; then
|
||||
cd ${ARCH}/RHEL-8-001
|
||||
createrepo_c . &> /dev/null
|
||||
else
|
||||
echo "Unable to run createrepo on latest/${ARCH}"
|
||||
fi
|
||||
popd
|
||||
done
|
||||
70
roles/batcave/files/rhel8-sync
Normal file
70
roles/batcave/files/rhel8-sync
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
#!/bin/sh
|
||||
|
||||
##
|
||||
# Setup the directories we will sync down to.
|
||||
RHEL8SYNCDIR=/srv/web/repo/rhel/rhel8
|
||||
SYNCDIR=/var/cache/reposync/
|
||||
|
||||
##
|
||||
# Need to download the metadata and the modular data
|
||||
DNFOPTS='--download-metadata -m --setopt=module_platform_id="platform:el8"'
|
||||
# Need to set the arches we are going to sync down
|
||||
ARCHES="aarch64 ppc64le s390x x86_64"
|
||||
|
||||
|
||||
##
|
||||
# Do the deeds
|
||||
cd ${RHEL8SYNCDIR}
|
||||
|
||||
# Make the sync dir where dnf will drop stuff. This disappeared once
|
||||
# and caused the script to die so we do this explicitely now.
|
||||
mkdir -vp ${SYNCDIR}
|
||||
|
||||
##
|
||||
## Sync Out RHEL-8 for appropriate arches
|
||||
for A in ${ARCHES}; do
|
||||
|
||||
# In case we have a bad sync from before, make sure the old sync
|
||||
# data is destroyed.
|
||||
rm -rf ${SYNCDIR}/${A}
|
||||
|
||||
# Configuration files for yum4 should fit this pattern.
|
||||
CONF=yum4-batcave-rhel8-${A}.conf
|
||||
|
||||
# Setup the repos you are going to sync out of the config file in
|
||||
# this pattern. This makes it easier for koji configs later
|
||||
REPOS="rhel-8-for-${A}-appstream-rpms rhel-8-for-${A}-baseos-rpms codeready-builder-for-rhel-8-${A}-rpms"
|
||||
|
||||
# We do a 2 level reposync because batcave is RHEL-7 and reposync
|
||||
# is what is default. However reposync does not pull in certain
|
||||
# data so we have to repeat this with dnf to get the metadata
|
||||
# needed for modularity.
|
||||
for R in ${REPOS}; do
|
||||
echo "Syncing ${R} for ${A}"
|
||||
reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A} &> /dev/null
|
||||
# Be verbose on errors.
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Something went wrong with reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -a noarch -p ${A}"
|
||||
fi
|
||||
# x86_64 is 'special' because we have to tell it for i686 also.
|
||||
if [[ ${A} == 'x86_64' ]]; then
|
||||
dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS} --repo=${R} -a ${A} -a noarch -a i686 -p ${A} &> /dev/null
|
||||
else
|
||||
dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS} --repo=${R} -a ${A} -a noarch -p ${A} &> /dev/null
|
||||
fi
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Something went wrong with dnf reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
### Here we put in syncs which do not fit in with the general do this
|
||||
### for all architectures. In this case we only copy down openstack
|
||||
### for x86_64 and source because we really only care about rabbitmq
|
||||
### We put this in an 'other' directory so that grobisplitter and any
|
||||
### other tools don't accidently pull it into their work.
|
||||
|
||||
OTHERCONF=yum4-batcave-rhel8-other.conf
|
||||
|
||||
reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-x86_64-openstack-16-rpms -a x86_64 -p other &> /dev/null
|
||||
reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-srpms-openstack-16-rpms --source -a x86_64 -p other &> /dev/null
|
||||
|
|
@ -325,6 +325,37 @@
|
|||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup rhel6 sync script.
|
||||
#
|
||||
|
||||
- name: setup rhel6 sync script
|
||||
copy: src=rhel6-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel6/rhel6-sync mode=0775
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
#
|
||||
# Setup rhel7 sync script.
|
||||
#
|
||||
|
||||
- name: setup rhel7 sync copy script
|
||||
copy: src=rhel7-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel7/rhel7-sync mode=0775
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
#
|
||||
# Setup rhel8 sync script.
|
||||
#
|
||||
|
||||
- name: setup rhel8 sync copy script
|
||||
copy: src=rhel8-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel8/rhel8-sync mode=0775
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# sync-rhn cron job
|
||||
#
|
||||
|
|
@ -335,6 +366,18 @@
|
|||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
|
||||
#
|
||||
# Setup centos sync script.
|
||||
#
|
||||
|
||||
- name: setup centos sync script
|
||||
copy: src=sync-centos dest=/mnt/fedora/app/fi-repo/centos/centos-8-sync mode=0775
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# sync-centos cron job
|
||||
#
|
||||
|
|
@ -345,6 +388,26 @@
|
|||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup rhel8-split script.
|
||||
#
|
||||
|
||||
- name: setup rhel8-split script
|
||||
copy: src=rhel8-split.sh dest=/usr/local/bin/rhel8-split.sh mode=0755
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup rhel8-split cron job
|
||||
#
|
||||
- name: setup rhel8-split cron
|
||||
copy: src=rhel8-split.cron dest=/etc/cron.d/rhel8-split.cron mode=0644
|
||||
tags:
|
||||
- batcave
|
||||
- config
|
||||
when: inventory_hostname.startswith('batcave01')
|
||||
|
||||
#
|
||||
# Setup web server config
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue