diff --git a/roles/batcave/files/centos-8-sync b/roles/batcave/files/centos-8-sync new file mode 100644 index 0000000000..209647b828 --- /dev/null +++ b/roles/batcave/files/centos-8-sync @@ -0,0 +1,19 @@ +#!/bin/sh + +RSYNC_DESTDIR=/mnt/fedora/app/fi-repo/centos/centos-8/ +RSYNC_MOD="centos-epel/centos/8.1.1911/" + +REPO_FROMDIR=/mnt/fedora/app/fi-repo/centos/centos-8/ + +mkdir -p ${RSYNC_DESTDIR} + +RSYNC='/usr/bin/rsync' +RS_OPT="-avSHP --numeric-ids" +#RS_DEADLY="--delete --delete-excluded --delete-delay --delay-updates" +RS_DEADLY="--delete --delete-delay --delay-updates" +CENT_EXCLUDES="--exclude=isos --exclude=kickstart --exclude=s390x" + +SERVER=centos-master01.iad2.fedoraproject.org + +${RSYNC} ${RS_OPT} ${RS_DEADLY} ${CENT_EXCLUDES} ${SERVER}::${RSYNC_MOD} ${RSYNC_DESTDIR} + diff --git a/roles/batcave/files/rhel6-sync b/roles/batcave/files/rhel6-sync new file mode 100644 index 0000000000..573bd04a47 --- /dev/null +++ b/roles/batcave/files/rhel6-sync @@ -0,0 +1,47 @@ +#!/bin/sh + +RHEL6SYNCDIR=/srv/web/repo/rhel/rhel6 +RHEL6CACHEDIR=/var/tmp/reposync-rhel6 + +cd $RHEL6SYNCDIR + + +mkdir -p ${RHEL6CACHEDIR} +for i in i686 x86_64 ppc64; do +mkdir -p ${RHEL6CACHEDIR}/${i} +done + +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686 +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-optional-i386 -p i386 -a i686 --metadata-path ${RHEL6CACHEDIR}/i686 +# 2019-09-17 RHEL does not have these anymore for i386 +#reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-ha-i386 -p i386 -a i686 +#reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-i386.conf -t -n --download-metadata --repoid rhel-6-server-lb-i386 -p i386 -a i686 + +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64 +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-optional-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64 +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-ha-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64 +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-x86_64.conf -t -n --download-metadata --repoid rhel-6-server-lb-x86_64 -p x86_64 --metadata-path ${RHEL6CACHEDIR}/x86_64 + +# +# rhel6 didn't have lb or ha repos for ppc64 +# +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-ppc64 -p ppc64 -a ppc64 --metadata-path ${RHEL6CACHEDIR}/ppc64 +reposync -c $RHEL6SYNCDIR/yum-lockbox-rhel6-ppc64.conf -t -n --download-metadata --repoid rhel-6-server-optional-ppc64 -p ppc64 -a ppc64 --metadata-path ${RHEL6CACHEDIR}/ppc64 + +for i in `ls -d /srv/web/repo/rhel/rhel6/i386/*` +do + cd $i + createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . +done + +for i in `ls -d /srv/web/repo/rhel/rhel6/x86_64/*` +do + cd $i + createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . +done + +for i in `ls -d /srv/web/repo/rhel/rhel6/ppc64/*` +do + cd $i + createrepo_c -q --update --compatibility --compress-type bz2 --workers 6 . +done diff --git a/roles/batcave/files/rhel7-sync b/roles/batcave/files/rhel7-sync new file mode 100644 index 0000000000..b271641749 --- /dev/null +++ b/roles/batcave/files/rhel7-sync @@ -0,0 +1,52 @@ +#!/bin/sh + +RHEL7SYNCDIR=/srv/web/repo/rhel/rhel7 +SYNCDIR=/var/cache/reposync/ +YUMDIR=/var/tmp/reposync-rhel7/ + +## Need to download the metadata and the modular data +ROPTS='--download-metadata -m' + +# s390 isnt working yet +ARCHES="ppc64le s390x x86_64" +#ARCHES="x86_64" + +mkdir -vp ${SYNCDIR} ${RHEL7SYNCDIR} ${YUMDIR} +for A in ${ARCHES}; do + mkdir -vp ${YUMDIR}/${A} +done + +cd $RHEL7SYNCDIR + +## +## Sync out RHEL-7 for appropriate arches +for A in ${ARCHES}; do + rm -rf ${SYNCDIR}/${A} + CONF=yum-batcave-rhel7-${A}.conf + REPOS="rhel-7-for-${A}-server-rpms rhel-7-ha-for-${A}-server-rpms rhel-7-extras-for-${A}-server-rpms rhel-7-optional-for-${A}-server-rpms rhel-7-devtools-for-${A}-server-rpms rhel-7-rhscl-for-${A}-server-rpms" +# REPOS="rhel-7-rhscl-for-${A}-server-rpms" + for R in ${REPOS}; do + rm -rf ${YUMDIR}/${R} + # Test to see if we have this repository in our tree + grep -q ${R} ${RHEL7SYNCDIR}/${CONF} + if [[ $? -eq 0 ]]; then + echo "Syncing ${R} for ${A}" + reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -p ${A} --metadata-path ${YUMDIR}/${A} + if [[ $? -ne 0 ]]; then + echo "Something went wrong with reposync -c ${RHEL7SYNCDIR}/${CONF} --repoid ${R} -a ${A} -a noarch -p ${A} --metadata-path ${YUMDIR}/${A}" + continue + fi + ## Try to create repo + pushd ${RHEL7SYNCDIR}/${A}/${R} &> /dev/null + createrepo_c -q --update --workers 6 . + if [[ $? -ne 0 ]]; then + echo "Something went wrong with createrepo ${A} ${R}" + continue + fi + popd &> /dev/null + fi + done +done + +#ostree pull --repo=/mnt/fedora/app/fi-repo/rhel/rhel7/atomic/rhel7 --mirror redhat +## This section is for all the trees which are not easily scriptable diff --git a/roles/batcave/files/rhel8-split.cron b/roles/batcave/files/rhel8-split.cron new file mode 100644 index 0000000000..5b9f3e3367 --- /dev/null +++ b/roles/batcave/files/rhel8-split.cron @@ -0,0 +1 @@ +03 09 * * * root /usr/local/bin/rhel8-split.sh diff --git a/roles/batcave/files/rhel8-split.sh b/roles/batcave/files/rhel8-split.sh new file mode 100644 index 0000000000..a42399e730 --- /dev/null +++ b/roles/batcave/files/rhel8-split.sh @@ -0,0 +1,99 @@ +#!/bin/bash +HOMEDIR=/mnt/fedora/app/fi-repo/rhel/rhel8 +BINDIR=/usr/local/bin + +ARCHES="aarch64 ppc64le s390x x86_64" +DATE=$(date -Ih | sed 's/+.*//') + +DATEDIR=${HOMEDIR}/koji/${DATE} + +if [ -d ${DATEDIR} ]; then + echo "Directory already exists. Please remove or fix" + exit +else +mkdir -p ${DATEDIR} +fi + +for ARCH in ${ARCHES}; do + # The archdir is where we daily download updates for rhel8 + ARCHDIR=${HOMEDIR}/${ARCH} + if [ ! -d ${ARCHDIR} ]; then + echo "Unable to find ${ARCHDIR}" + exit + fi + + # We consolidate all of the default repositories and remerge them + # in a daily tree. This allows us to point koji at a particular + # day if we have specific build concerns. + OUTDIR=${DATEDIR}/${ARCH} + mkdir -p ${OUTDIR} + if [ ! -d ${OUTDIR} ]; then + echo "Unable to find ${ARCHDIR}" + exit + else + cd ${OUTDIR} + fi + + # Begin splitting the various packages into their subtrees + ${BINDIR}/splitter.py --action hardlink --target RHEL-8-001 ${ARCHDIR}/rhel-8-for-${ARCH}-baseos-rpms/ --only-defaults &> /dev/null + if [ $? -ne 0 ]; then + echo "splitter ${ARCH} baseos failed" + exit + fi + ${BINDIR}/splitter.py --action hardlink --target RHEL-8-002 ${ARCHDIR}/rhel-8-for-${ARCH}-appstream-rpms/ --only-defaults &> /dev/null + if [ $? -ne 0 ]; then + echo "splitter ${ARCH} appstream failed" + exit + fi + ${BINDIR}/splitter.py --action hardlink --target RHEL-8-003 ${ARCHDIR}/codeready-builder-for-rhel-8-${ARCH}-rpms/ &> /dev/null + if [ $? -ne 0 ]; then + echo "splitter ${ARCH} codeready failed" + exit + fi + + # Copy the various module trees into RHEL-8-001 where we want them + # to work. + echo "Moving data to ${ARCH}/RHEL-8-001" + cp -anlr RHEL-8-002/* RHEL-8-001 + cp -anlr RHEL-8-003/* RHEL-8-001 + # Go into the main tree + pushd RHEL-8-001 + + # Mergerepo didn't work so lets just createrepo in the top directory. + createrepo_c . &> /dev/null + popd + + # Cleanup the trash + rm -rf RHEL-8-002 RHEL-8-003 +#loop to the next +done + +## Set up the builds so they are pointing to the last working version +cd ${HOMEDIR}/koji/ +if [[ -e staged ]]; then + if [[ -h staged ]]; then + rm -f staged + else + echo "Unable to remove staged. it is not a symbolic link" + exit + fi +else + echo "No staged link found" +fi + +echo "Linking ${DATE} to staged" +ln -s ${DATE} staged + + +for ARCH in ${ARCHES}; do + pushd latest/ + mkdir -p ${ARCH} + dnf --disablerepo=\* --enablerepo=RHEL-8-001 --repofrompath=RHEL-8-001,https://infrastructure.fedoraproject.org/repo/rhel/rhel8/koji/staged/${ARCH}/RHEL-8-001/ reposync -a ${ARCH} -a noarch -p ${ARCH} --newest --delete &> /dev/null + if [[ $? -eq 0 ]]; then + cd ${ARCH}/RHEL-8-001 + createrepo_c . &> /dev/null + else + echo "Unable to run createrepo on latest/${ARCH}" + fi + popd +done diff --git a/roles/batcave/files/rhel8-sync b/roles/batcave/files/rhel8-sync new file mode 100644 index 0000000000..95bf77e7f1 --- /dev/null +++ b/roles/batcave/files/rhel8-sync @@ -0,0 +1,70 @@ +#!/bin/sh + +## +# Setup the directories we will sync down to. +RHEL8SYNCDIR=/srv/web/repo/rhel/rhel8 +SYNCDIR=/var/cache/reposync/ + +## +# Need to download the metadata and the modular data +DNFOPTS='--download-metadata -m --setopt=module_platform_id="platform:el8"' +# Need to set the arches we are going to sync down +ARCHES="aarch64 ppc64le s390x x86_64" + + +## +# Do the deeds +cd ${RHEL8SYNCDIR} + +# Make the sync dir where dnf will drop stuff. This disappeared once +# and caused the script to die so we do this explicitely now. +mkdir -vp ${SYNCDIR} + +## +## Sync Out RHEL-8 for appropriate arches +for A in ${ARCHES}; do + + # In case we have a bad sync from before, make sure the old sync + # data is destroyed. + rm -rf ${SYNCDIR}/${A} + + # Configuration files for yum4 should fit this pattern. + CONF=yum4-batcave-rhel8-${A}.conf + + # Setup the repos you are going to sync out of the config file in + # this pattern. This makes it easier for koji configs later + REPOS="rhel-8-for-${A}-appstream-rpms rhel-8-for-${A}-baseos-rpms codeready-builder-for-rhel-8-${A}-rpms" + + # We do a 2 level reposync because batcave is RHEL-7 and reposync + # is what is default. However reposync does not pull in certain + # data so we have to repeat this with dnf to get the metadata + # needed for modularity. + for R in ${REPOS}; do + echo "Syncing ${R} for ${A}" + reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A} &> /dev/null + # Be verbose on errors. + if [[ $? -ne 0 ]]; then + echo "Something went wrong with reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -a noarch -p ${A}" + fi + # x86_64 is 'special' because we have to tell it for i686 also. + if [[ ${A} == 'x86_64' ]]; then + dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS} --repo=${R} -a ${A} -a noarch -a i686 -p ${A} &> /dev/null + else + dnf reposync -c ${RHEL8SYNCDIR}/${CONF} ${DNFOPTS} --repo=${R} -a ${A} -a noarch -p ${A} &> /dev/null + fi + if [[ $? -ne 0 ]]; then + echo "Something went wrong with dnf reposync -c ${RHEL8SYNCDIR}/${CONF} -r ${R} -a ${A} -p ${A}" + fi + done +done + +### Here we put in syncs which do not fit in with the general do this +### for all architectures. In this case we only copy down openstack +### for x86_64 and source because we really only care about rabbitmq +### We put this in an 'other' directory so that grobisplitter and any +### other tools don't accidently pull it into their work. + +OTHERCONF=yum4-batcave-rhel8-other.conf + +reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-x86_64-openstack-16-rpms -a x86_64 -p other &> /dev/null +reposync -c ${RHEL8SYNCDIR}/${OTHERCONF} -r rhel-8-srpms-openstack-16-rpms --source -a x86_64 -p other &> /dev/null diff --git a/roles/batcave/tasks/main.yml b/roles/batcave/tasks/main.yml index efb8933e7c..09b204ecd0 100644 --- a/roles/batcave/tasks/main.yml +++ b/roles/batcave/tasks/main.yml @@ -325,6 +325,37 @@ - config when: inventory_hostname.startswith('batcave01') +# +# Setup rhel6 sync script. +# + +- name: setup rhel6 sync script + copy: src=rhel6-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel6/rhel6-sync mode=0775 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') +# +# Setup rhel7 sync script. +# + +- name: setup rhel7 sync copy script + copy: src=rhel7-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel7/rhel7-sync mode=0775 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') +# +# Setup rhel8 sync script. +# + +- name: setup rhel8 sync copy script + copy: src=rhel8-sync dest=/mnt/fedora/app/fi-repo/rhel/rhel8/rhel8-sync mode=0775 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') + # # sync-rhn cron job # @@ -335,6 +366,18 @@ - config when: inventory_hostname.startswith('batcave01') + +# +# Setup centos sync script. +# + +- name: setup centos sync script + copy: src=sync-centos dest=/mnt/fedora/app/fi-repo/centos/centos-8-sync mode=0775 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') + # # sync-centos cron job # @@ -345,6 +388,26 @@ - config when: inventory_hostname.startswith('batcave01') +# +# Setup rhel8-split script. +# + +- name: setup rhel8-split script + copy: src=rhel8-split.sh dest=/usr/local/bin/rhel8-split.sh mode=0755 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') + +# +# Setup rhel8-split cron job +# +- name: setup rhel8-split cron + copy: src=rhel8-split.cron dest=/etc/cron.d/rhel8-split.cron mode=0644 + tags: + - batcave + - config + when: inventory_hostname.startswith('batcave01') # # Setup web server config