diff --git a/files/scripts/linuxsystemroles-logs-clean b/files/scripts/linuxsystemroles-logs-clean new file mode 100755 index 0000000000..ca8f8fcaf8 --- /dev/null +++ b/files/scripts/linuxsystemroles-logs-clean @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +AGE_IN_DAYS=${AGE_IN_DAYS:-183} +newest_date=$(date --date="${AGE_IN_DAYS} days ago" +%Y%m%d) + +if [ -z "${1:-}" ]; then + echo "Need a directory to pushd in" >&2 + exit 1 +fi + +pushd "${1}" + +find -maxdepth 1 -type d | while read dir; do + # Case: linux-system-roles-certificate-pull-linux-system-roles_certificate-80-4f880f7-rhel-x-20210305-152227 + if [[ "${dir}" =~ ^./linux-system-roles-[[:alnum:]._-]+-pull-linux-system-roles_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)-([[:alnum:]._-]+)-([[:digit:]]+)-([[:digit:]]+)$ ]]; then + echo "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${dir}" + # Case: lsr-citool_bootloader-19-0f14842_20220104-080416 + elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:digit:]]+)-([[:digit:]]+)$ ]]; then + echo "${BASH_REMATCH[1]}" "unknown" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${dir}" + # Case: lsr-citool_certificate-132-212741b_RHEL-9.1.0-20220814.1_20220818-223408 + elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:alnum:]._-]+)_([[:digit:]]+)-([[:digit:]]+)$ ]]; then + echo "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}" "${BASH_REMATCH[5]}" "${dir}" + # Case: lsr-citool_network-509-82dd06b_RHEL-6.10-updates-20201110.17 + elif [[ "${dir}" =~ ^./lsr-citool_([[:alnum:]._-]+)-[[:digit:]]+-([[:xdigit:]]+|HEAD)_([[:alnum:]._-]+)$ ]]; then + tmod="$(stat -c %Y "${dir}")" + tmod="$(date --date="@${tmod}" +'%Y%m%d %H%M%S')" + echo "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}" "${tmod}" "${dir}" + # Case: lsr-citool_* (artifacts of early lsr-citool development) + elif [[ "${dir}" =~ ^./lsr-citool_.*$ ]]; then + tmod="$(stat -c %Y "${dir}")" + tmod="$(date --date="@${tmod}" +'%Y%m%d %H%M%S')" + echo unknown unknown "${tmod}" "${dir}" + fi +done | sort -n -r | while read role image date time dir; do + # `sort -n -r` ensures that the most recent log for $role $image is on the top + if [[ "${role}" = unknown && "${date}" -lt "${newest_date}" ]]; then + # Let not artifacts of early stage of lsr-citool development to rot forever + rm -rf "${dir}" + continue + fi + latest_file=".latest_${role}_${image}" + if [[ -f "${latest_file}" ]]; then + # Latest log for $role $image seen already + if [[ "${date}" -lt "${newest_date}" ]]; then + rm -rf "${dir}" + fi + else + echo keeping ${role} ${image} ${date} ${time} ${dir} + echo "${dir}" > "${latest_file}" + fi +done +rm -rf .latest_* + +popd diff --git a/files/scripts/linuxsystemroles-logs-clean.test b/files/scripts/linuxsystemroles-logs-clean.test new file mode 100755 index 0000000000..be370b6ace --- /dev/null +++ b/files/scripts/linuxsystemroles-logs-clean.test @@ -0,0 +1,405 @@ +#!/bin/bash +# +# Test `linuxsystemroles-logs-clean` script. +# +# Usage: ./linuxsystemroles-logs-clean.test +# +# Expects `linuxsystemroles-logs-clean` script in the same directory. + +set -euo pipefail + +: <<_EOF_ + +Roles (as found in log names at /srv/pub/alt/linuxsystemroles/logs): + "ad_integration" + "bootloader" + "certificate" + "ci-test" + "ci-testing" + "cockpit" + "crypto_policies" + "firewall" + "ha_cluster" + "kdump" + "kernel_settings" + "logging" + "metrics" + "mssql" + "nbde_client" + "nbde_server" + "network" + "podman" + "postfix" + "postgresql" + "rhc" + "selinux" + "ssh" + "storage" + "timesync" + "tlog" + "tuned" + "vpn" + +Images (as found in log names at /srv/pub/alt/linuxsystemroles/logs): + "centos-6" + "centos-7" + "centos-8" + "fedora-33" + "fedora-34" + "fedora-35" + "rhel-6" + "rhel-7" + "rhel-8" + "rhel-8-y" + "rhel-x" + "CentOS-7-latest" + "CentOS-Stream-8" + "Fedora-36" + "Fedora-37" + "RHEL-6.10-updates-20201110.17" + "RHEL-7.9-updates-20221012.4" + "RHEL-8.8.0-20221211.0" + "RHEL-9.2.0-20221212.0" + +_EOF_ + +ME="$(basename ${0})" +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +NOW="$(date +'%Y-%m-%d %H:%M:%S')" +TEMPDIR="$(mktemp -d /var/tmp/${ME}.XXXXXXXXXX)" +STATUS=0 + +trap "rm -rf ${TEMPDIR}" ABRT EXIT HUP INT QUIT TERM + +## +# failure - report test's failure and set STATUS to 1 +# +# Parameters: +# $1 - message +# +function failure() { + echo $* >&2 + STATUS=1 +} + +## +# error - report an error and exit with 2 +# +# Parameters: +# $1 - message +# +function error() { + echo $* >&2 + exit 2 +} + +## +# log_1_name - generate a name of a directory with logs +# +# Parameters: +# $1 - role name +# $2 - pull request number +# $3 - commit hash +# $4 - image name +# $5 - date +# $6 - time +# +function log_1_name() { + echo "linux-system-roles-${1}-pull-linux-system-roles_${1}-${2}-${3}-${4}-${5}-${6}" +} + +## +# log_2_name - see log_1_name +# +function log_2_name() { + if [[ -z "${4:-}" ]]; then + echo "lsr-citool_${1}-${2}-${3}_${5}-${6}" + elif [[ -z "${5:-}" || -z "${6:-}" ]]; then + echo "lsr-citool_${1}-${2}-${3}_${4}" + else + echo "lsr-citool_${1}-${2}-${3}_${4}_${5}-${6}" + fi +} + +## +# log_3_name - see log_1_name +# +function log_3_name() { + # Produced by early stage of development of lsr-citool + echo "lsr-citool_ci-tt" +} + +## +# log_4_name - see log_1_name +# +function log_4_name() { + # Produced by early stage of development of lsr-citool + echo "lsr-citool_repoName-7-54sd5fsd_${5}-${6}" +} + +## +# stay - decides whether log should stay or be removed (auxiliary function +# used by prep_test) +# +# Parameters: +# $1 - days ago divided by 10 (integer division) +# +function stay() { + if [[ ${1:-0} -ge 18 ]]; then + echo 0 + else + echo 1 + fi +} + +## +# stay3 - decides which group of logs should stay (see prep_test) +# +# Parameters: +# $1 - group A days ago divided by 10 (integer division) +# $2 - group B days ago divided by 10 (integer division) +# $3 - group C days ago divided by 10 (integer division) +# $4 - selector +# +function stay3() { + local A="$(stay "${1:-}")" + local B="$(stay "${2:-}")" + local C="$(stay "${3:-}")" + + # It is assumed that group A is younger than group B is younger than group C + # (see prep_test). Thus, A should stay if all groups are deletion candidates + if [[ "${A}${B}${C}" = "000" ]]; then + A=1 + fi + + case "${4:-}" in + A) echo "${A}";; + B) echo "${B}";; + C) echo "${C}";; + *) error "stay3: Invalid selector (${4:-})";; + esac +} + +## +# create_log - create a log +# +# Options: +# -n - do not include time stamp into log name +# +# Parameters: +# $1 - log name generator +# $2 - role name +# $3 - pull request number +# $4 - commit hash +# $5 - image name +# $6 - time ago (___) +# $7 - stay flag (1 - log should stay, 0 - log should be removed) +# +function create_log() { + local INCLUDE_STAMP=1 + local LOG_STAMP="" + local LOG_DATE="" + local LOG_TIME="" + local LOG_NAME="" + local TEMP="" + + if [[ "X${1:-}" = X-n ]]; then + INCLUDE_STAMP=0 + shift + fi + + if [[ "${6:-}" =~ ^([[:digit:]]+)_([[:digit:]]+)_([[:digit:]]+)_([[:digit:]]+)$ ]]; then + if [[ "${BASH_REMATCH[1]}" -gt 0 ]]; then + TEMP+="${BASH_REMATCH[1]} days ago " + fi + if [[ "${BASH_REMATCH[2]}" -gt 0 ]]; then + TEMP+="${BASH_REMATCH[2]} hours ago " + fi + if [[ "${BASH_REMATCH[3]}" -gt 0 ]]; then + TEMP+="${BASH_REMATCH[3]} minutes ago " + fi + if [[ "${BASH_REMATCH[4]}" -gt 0 ]]; then + TEMP+="${BASH_REMATCH[4]} seconds ago " + fi + LOG_STAMP="$(date --date="${NOW} ${TEMP}" +'%Y-%m-%d %H:%M:%S')" + LOG_DATE="$(date --date="${LOG_STAMP}" +%Y%m%d)" + LOG_TIME="$(date --date="${LOG_STAMP}" +%H%M%S)" + else + error "create_log: Invalid time ago (${6:-})" + fi + + if [[ ${INCLUDE_STAMP} -eq 1 ]]; then + LOG_NAME="$(${1} "${2}" "${3}" "${4}" "${5}" ${LOG_DATE} ${LOG_TIME})" + else + LOG_NAME="$(${1} "${2}" "${3}" "${4}" "${5}" "" "")" + fi + + mkdir -p ${TEMPDIR}/logs/${LOG_NAME} + echo log > ${TEMPDIR}/logs/${LOG_NAME}/log + + touch --date="${LOG_STAMP}" ${TEMPDIR}/logs/${LOG_NAME}/log ${TEMPDIR}/logs/${LOG_NAME} + + echo ${7} ${LOG_NAME} >> ${TEMPDIR}/assertions +} + +## +# prep_test - prepare a test directory +# +# Parameters: +# $1 - A-group age (1 - young, 0 - old) +# $2 - B-group age (1 - young, 0 - old) +# $3 - C-group age (1 - young, 0 - old) +# $4 - D-group age (1 - young, 0 - old) +# +function prep_test() { + local A="" + local B="" + local C="" + local D="" + + if [[ ${1:-1} -eq 0 ]]; then + A=20 + fi + if [[ ${2:-1} -eq 0 ]]; then + B=20 + fi + if [[ ${3:-1} -eq 0 ]]; then + C=20 + fi + if [[ ${4:-1} -eq 0 ]]; then + D=20 + fi + + create_log log_1_name certificate 0 HEAD centos-8 ${A}5_0_0_0 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name certificate 0 HEAD rhel-8-y ${A}5_0_0_5 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name certificate 0 HEAD rhel-x ${A}5_0_0_10 $(stay3 "${A}" "${B}" "${C}" A) + + create_log log_1_name certificate 1 3dbef8b centos-8 ${B}5_0_5_0 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name certificate 1 3dbef8b rhel-8-y ${B}5_0_5_5 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name certificate 1 3dbef8b rhel-x ${B}5_0_5_10 $(stay3 "${A}" "${B}" "${C}" B) + + create_log log_1_name certificate 15 0c2a97e centos-8 ${C}5_0_10_0 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name certificate 15 0c2a97e rhel-8-y ${C}5_0_10_5 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name certificate 15 0c2a97e rhel-x ${C}5_0_10_10 $(stay3 "${A}" "${B}" "${C}" C) + + create_log log_1_name network 127 2936b72 fedora-35 ${A}5_0_1_0 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name network 127 2936b72 rhel-6 ${A}5_0_1_5 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name network 127 2936b72 rhel-x ${A}5_0_1_10 $(stay3 "${A}" "${B}" "${C}" A) + + create_log log_1_name network 136 5480c71 fedora-35 ${B}5_0_6_0 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name network 136 5480c71 rhel-6 ${B}5_0_6_5 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name network 136 5480c71 rhel-x ${B}5_0_6_10 $(stay3 "${A}" "${B}" "${C}" B) + + create_log log_1_name network 201 c09573b fedora-35 ${C}5_0_11_0 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name network 201 c09573b rhel-6 ${C}5_0_11_5 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name network 201 c09573b rhel-x ${C}5_0_11_10 $(stay3 "${A}" "${B}" "${C}" C) + + create_log log_1_name timesync 17 ec8e4f4 centos-6 ${A}5_0_2_0 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name timesync 17 ec8e4f4 fedora-34 ${A}5_0_2_5 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_1_name timesync 17 ec8e4f4 rhel-7 ${A}5_0_2_10 $(stay3 "${A}" "${B}" "${C}" A) + + create_log log_1_name timesync 22 41265d0 centos-6 ${B}5_0_7_0 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name timesync 22 41265d0 fedora-34 ${B}5_0_7_5 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_1_name timesync 22 41265d0 rhel-7 ${B}5_0_7_10 $(stay3 "${A}" "${B}" "${C}" B) + + create_log log_1_name timesync 99 f5dae11 centos-6 ${C}5_0_12_0 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name timesync 99 f5dae11 fedora-34 ${C}5_0_12_5 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_1_name timesync 99 f5dae11 rhel-7 ${C}5_0_12_10 $(stay3 "${A}" "${B}" "${C}" C) + + create_log log_2_name ad_integration 1 aeae773 "" ${A}5_0_3_0 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name ad_integration 1 aeae773 CentOS-7-latest ${A}5_0_3_5 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name ad_integration 1 aeae773 CentOS-Stream-8 ${A}5_0_3_10 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name ad_integration 1 aeae773 Fedora-37 ${A}5_0_3_15 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name ad_integration 1 aeae773 RHEL-6.10-updates-20201110.17 ${A}5_0_3_20 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name ad_integration 1 aeae773 RHEL-9.2.0-20221212.0 ${A}5_0_3_25 $(stay3 "${A}" "${B}" "${C}" A) + + create_log log_2_name ad_integration 11 f1f12e2 "" ${B}5_0_8_0 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name ad_integration 11 f1f12e2 CentOS-7-latest ${B}5_0_8_5 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name ad_integration 11 f1f12e2 CentOS-Stream-8 ${B}5_0_8_10 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name ad_integration 11 f1f12e2 Fedora-37 ${B}5_0_8_15 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name ad_integration 11 f1f12e2 RHEL-6.10-updates-20201110.17 ${B}5_0_8_20 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name ad_integration 11 f1f12e2 RHEL-9.2.0-20221212.0 ${B}5_0_8_25 $(stay3 "${A}" "${B}" "${C}" B) + + create_log log_2_name ad_integration 32 1148efe "" ${C}5_0_13_0 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name ad_integration 32 1148efe CentOS-7-latest ${C}5_0_13_5 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name ad_integration 32 1148efe CentOS-Stream-8 ${C}5_0_13_10 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name ad_integration 32 1148efe Fedora-37 ${C}5_0_13_15 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name ad_integration 32 1148efe RHEL-6.10-updates-20201110.17 ${C}5_0_13_20 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name ad_integration 32 1148efe RHEL-9.2.0-20221212.0 ${C}5_0_13_25 $(stay3 "${A}" "${B}" "${C}" C) + + create_log log_2_name bootloader 2 264569a "" ${A}5_0_4_0 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name bootloader 2 264569a CentOS-Stream-8 ${A}5_0_4_5 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name bootloader 2 264569a Fedora-36 ${A}5_0_4_10 $(stay3 "${A}" "${B}" "${C}" A) + create_log log_2_name bootloader 2 264569a RHEL-8.8.0-20221211.0 ${A}5_0_4_15 $(stay3 "${A}" "${B}" "${C}" A) + create_log -n log_2_name bootloader 2 264569a RHEL-9.2.0-20221212.0 ${A}5_0_4_20 $(stay3 "${A}" "${B}" "${C}" A) + + create_log log_2_name bootloader 21 d1f589b "" ${B}5_0_9_0 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name bootloader 21 d1f589b CentOS-Stream-8 ${B}5_0_9_5 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name bootloader 21 d1f589b Fedora-36 ${B}5_0_9_10 $(stay3 "${A}" "${B}" "${C}" B) + create_log log_2_name bootloader 21 d1f589b RHEL-8.8.0-20221211.0 ${B}5_0_9_15 $(stay3 "${A}" "${B}" "${C}" B) + create_log -n log_2_name bootloader 21 d1f589b RHEL-9.2.0-20221212.0 ${B}5_0_9_20 $(stay3 "${A}" "${B}" "${C}" B) + + create_log log_2_name bootloader 82 bfa1e33 "" ${C}5_0_14_0 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name bootloader 82 bfa1e33 CentOS-Stream-8 ${C}5_0_14_5 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name bootloader 82 bfa1e33 Fedora-36 ${C}5_0_14_10 $(stay3 "${A}" "${B}" "${C}" C) + create_log log_2_name bootloader 82 bfa1e33 RHEL-8.8.0-20221211.0 ${C}5_0_14_15 $(stay3 "${A}" "${B}" "${C}" C) + create_log -n log_2_name bootloader 82 bfa1e33 RHEL-9.2.0-20221212.0 ${C}5_0_14_20 $(stay3 "${A}" "${B}" "${C}" C) + + create_log -n log_3_name "" "" "" "" ${D}5_0_30_0 $(stay ${D}) + create_log log_4_name "" "" "" "" ${D}5_0_30_5 $(stay ${D}) +} + +## +# verify_results - verify the test results +# +function verify_results() { + while read PRESENT LOG_NAME; do + if [[ "${PRESENT}" -eq 0 && -d ${TEMPDIR}/logs/${LOG_NAME} ]]; then + failure "[FAIL] Log ${LOG_NAME} should be removed." + elif [[ "${PRESENT}" -eq 1 && ! -d ${TEMPDIR}/logs/${LOG_NAME} ]]; then + failure "[FAIL] Log ${LOG_NAME} should be present." + fi + done < ${TEMPDIR}/assertions +} + +## +# clean_test - remove test assets +# +function clean_test() { + rm -rf ${TEMPDIR}/logs ${TEMPDIR}/assertions +} + +## +# test_log_pruning - test log pruning script +# +# Parameters: +# see prep_test +# +function test_log_pruning() { + prep_test "$@" + ${HERE}/linuxsystemroles-logs-clean ${TEMPDIR}/logs >/dev/null + verify_results + clean_test +} + +test_log_pruning 0 0 0 0 +test_log_pruning 0 0 0 1 +test_log_pruning 0 0 1 0 +test_log_pruning 0 0 1 1 + +test_log_pruning 0 1 0 0 +test_log_pruning 0 1 0 1 +test_log_pruning 0 1 1 0 +test_log_pruning 0 1 1 1 + +test_log_pruning 1 0 0 0 +test_log_pruning 1 0 0 1 +test_log_pruning 1 0 1 0 +test_log_pruning 1 0 1 1 + +test_log_pruning 1 1 0 0 +test_log_pruning 1 1 0 1 +test_log_pruning 1 1 1 0 +test_log_pruning 1 1 1 1 + +[[ ${STATUS} -eq 0 ]] diff --git a/playbooks/groups/secondary.yml b/playbooks/groups/secondary.yml index 82505432c4..fb459bdb5e 100644 --- a/playbooks/groups/secondary.yml +++ b/playbooks/groups/secondary.yml @@ -68,10 +68,18 @@ - name: add cron script to update fullfiletimelist copy: src="{{ files }}/scripts/update-fullfiletimelist" dest=/usr/local/bin/update-fullfiletimelist mode=0755 + - name: add cron script to prune old logs at /srv/pub/alt/linuxsystemroles/logs + copy: src="{{ files }}/scripts/linuxsystemroles-logs-clean" dest=/usr/local/bin/linuxsystemroles-logs-clean mode=0755 + - name: Update fullfiletimelist job cron: name="update-fullfiletimelist" hour="*/2" minute="55" user="root" job="/usr/local/bin/lock-wrapper update-fullfiletimelist '/usr/local/bin/update-fullfiletimelist -l /tmp/update-fullfiletimelist.lock -t /srv/pub alt'" cron_file=update-fullfiletimelist + - name: Prune old logs at /srv/pub/alt/linuxsystemroles/logs + cron: name="linuxsystemroles-logs-clean" hour="0" minute="15" user="root" + job="/usr/local/bin/linuxsystemroles-logs-clean /srv/pub/alt/linuxsystemroles/logs" + cron_file=linuxsystemroles-logs-clean + handlers: - import_tasks: "{{ handlers_path }}/restart_services.yml"