diff --git a/pulp-cleanup/build/Jenkinsfile b/pulp-cleanup/build/Jenkinsfile new file mode 100644 index 000000000..dd6241c71 --- /dev/null +++ b/pulp-cleanup/build/Jenkinsfile @@ -0,0 +1,68 @@ +import groovy.transform.Field + +@Field String pulpServerUrl = "https://pulp.front.sepia.ceph.com" + + +pipeline { + agent { + label 'small' + } + + options { + timeout(time: 1, unit: 'HOURS') + timestamps() + } + + stages { + stage('Checkout') { + steps { + checkout scm + } + } + + stage('Setup Pulp Client') { + steps { + script { + withCredentials([ + usernamePassword( + credentialsId: 'pulp-admin-auth', + usernameVariable: 'PULP_USERNAME', + passwordVariable: 'PULP_PASSWORD' + ) + ]) { + echo "Setting up Pulp Client..." + sh """#!/bin/bash -ex + export PULP_SERVER_URL="${pulpServerUrl}" && ./scripts/setup_pulp.sh + """ + } + } + } + } + + stage('Setup yq') { + steps { + script { + echo "Setting up yq..." + sh """#!/bin/bash -ex + source ./scripts/setup_yq.sh + """ + } + } + } + + stage('Execute Pulp Cleanup for Ceph') { + when { + expression { + params.CEPH + } + } + steps { + script { + def project = "ceph" + echo "Starting Pulp Cleanup for project: ${project}" + sh "./pulp-cleanup/build/cleanup.sh ${project}" + } + } + } + } +} diff --git a/pulp-cleanup/build/cleanup.sh b/pulp-cleanup/build/cleanup.sh new file mode 100755 index 000000000..695b5211a --- /dev/null +++ b/pulp-cleanup/build/cleanup.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +# Purge stale Pulp repositories and distributions per purge-policy.yaml. + +set -euo pipefail + +readonly PROJECT="$1" +readonly PURGE_POLICY_FILE="${WORKSPACE}/scripts/purge-policy.yaml" + +readonly LABEL="ref" +readonly PULP_LIST_LIMIT=1000 +readonly PULP_TYPES=(rpm deb) +readonly PULP_RESOURCES=(distribution repository) + +# Default protection time in hours +readonly PROTECTION_TIME=24 + +log() { + echo "[pulp_cleanup] $*" >&2 +} + +get_policy_refs() { + local project="$1" + local label="$2" + + yq -r ".${project}.${label} | keys[]" "${PURGE_POLICY_FILE}" +} + +read_default_purge_policy() { + local project="$1" + local -n _days="$2" + local policy_line + + policy_line=$( + yq -r \ + ".${project}.default | [(.days // \"\")] | @tsv" \ + "${PURGE_POLICY_FILE}" + ) + IFS=$'\t' read -r _days <<< "${policy_line}" +} + +fetch_pulp_resource_json() { + local type="$1" + local resource="$2" + local project="$3" + local label_select="$4" + + pulp "${type}" "${resource}" list \ + --limit "${PULP_LIST_LIMIT}" \ + --ordering '-pulp_created' \ + --label-select project="${project}","${label_select}" \ + 2>&1 | sed -n '/^\[/,$p' +} + +filter_resource_names_by_age() { + local pulp_json="$1" + local days="$2" + + if [ -z "${days}" ]; then + echo "${pulp_json}" | jq -r '.[].name' + return 0 + fi + + echo "${pulp_json}" | jq -r --arg days "${days}" ' + .[] + | select( + (.pulp_created | split(".")[0] + "Z" | fromdateiso8601) + < (now - ($days | tonumber) * 86400) + ) + | .name + ' +} + +list_stale_pulp_resources() { + local type="$1" + local resource="$2" + local project="$3" + local label_select="$4" + local days="$5" + local pulp_json resource_list + + pulp_json=$( + fetch_pulp_resource_json \ + "${type}" "${resource}" "${project}" "${label_select}" + ) + resource_list=$( + filter_resource_names_by_age "${pulp_json}" "${days}" + ) + echo "${resource_list}" +} + +destroy_pulp_resources() { + local type="$1" + local resource="$2" + local resource_list="$3" + local name + + if [ -z "${resource_list}" ]; then + return 0 + fi + + while IFS= read -r name; do + [ -z "${name}" ] && continue + log "Destroying ${type} ${resource}: ${name}" + if ! pulp "${type}" "${resource}" destroy \ + --name "${name}"; then + log "ERROR: Failed to destroy ${type} ${resource}: ${name}" + return 1 + fi + done <<< "${resource_list}" +} + +collect_stale_resources_for_ref() { + local project="$1" + local label_select="$2" + local days="$3" + local type resource key + + declare -n _stale_resources="$4" + + for type in "${PULP_TYPES[@]}"; do + for resource in "${PULP_RESOURCES[@]}"; do + key="${type}_${resource}" + _stale_resources[$key]=$( + list_stale_pulp_resources \ + "${type}" "${resource}" "${project}" \ + "${label_select}" "${days}" + ) + log "${key}: ${_stale_resources[$key]:-}" + done + done +} + +purge_resources() { + local project="$1" + local label_select="$2" + local days="$3" + local type resource + + declare -A stale_resources=() + log "Purging days=${days:-unset} label_select=${label_select}" + collect_stale_resources_for_ref "${project}" "${label_select}" "${days}" \ + stale_resources + + log "stale_resources: ${stale_resources[@]}" + + for type in "${PULP_TYPES[@]}"; do + destroy_pulp_resources "${type}" "distribution" \ + "${stale_resources[${type}_distribution]:-}" + done + for type in "${PULP_TYPES[@]}"; do + destroy_pulp_resources "${type}" "repository" \ + "${stale_resources[${type}_repository]:-}" + done +} + +run_package_cleanup() { + local project="$1" + local label="$2" + local days + + mapfile -t policy_refs < <(get_policy_refs "${project}" "${label}") + _label_select=$( + printf "${label}!=%s\n" "${policy_refs[@]}" | paste -sd, - + ) + log "label_select: ${_label_select}" + + read_default_purge_policy "${project}" days + log "days: ${days}" + + purge_resources "${project}" "${_label_select}" "${days}" +} + +run_orphan_cleanup() { + log "Applying orphan content purge policy" + if ! pulp orphan cleanup \ + --protection-time "${PROTECTION_TIME}"; then + log "ERROR: Failed to run orphan cleanup" + return 1 + fi + log "Orphan cleanup completed" +} + +log "Cleaning up project: ${PROJECT}" + +log "Running package cleanup" +run_package_cleanup "${PROJECT}" "${LABEL}" + +log "Running orphan cleanup" +run_orphan_cleanup diff --git a/pulp-cleanup/config/definitions/pulp-cleanup.yaml b/pulp-cleanup/config/definitions/pulp-cleanup.yaml new file mode 100644 index 000000000..4d12a95f9 --- /dev/null +++ b/pulp-cleanup/config/definitions/pulp-cleanup.yaml @@ -0,0 +1,37 @@ +- job: + name: pulp-cleanup-nightly + disabled: true + display-name: 'Pulp: Cleanup Nightly' + project-type: pipeline + description: > + Nightly parameterized cleanup of stale Teuthology Pulp resources. + defaults: global + + pipeline-scm: + scm: + - git: + url: https://github.com/ceph/ceph-build + branches: + - ${{CEPH_BUILD_BRANCH}} + shallow-clone: true + submodule: + disable: true + wipe-workspace: true + script-path: pulp-cleanup/build/Jenkinsfile + lightweight-checkout: true + do-not-fetch-tags: true + + triggers: + - timed: '@midnight' + + parameters: + - string: + name: CEPH_BUILD_BRANCH + description: 'Use the Jenkinsfile from this ceph-build branch' + default: 'main' + - bool: + name: CEPH + description: > + When true, clean up stale Pulp resources for the ceph project + per purge-policy.yaml. + default: true diff --git a/scripts/pulp_upload.sh b/scripts/pulp_upload.sh index 00c1c3627..a96d35fdf 100755 --- a/scripts/pulp_upload.sh +++ b/scripts/pulp_upload.sh @@ -25,7 +25,12 @@ source ./scripts/build_utils.sh export PATH="$HOME/.local/bin:$PATH" +# Pulp project name PULP_PROJECT="ceph" + +# Set purge policy file +PURGE_POLICY_FILE="${WORKSPACE}/scripts/purge-policy.yaml" + # Must match the base URL the Pulp client is configured with in setup_pulp.sh PULP_SERVER_URL="https://pulp.front.sepia.ceph.com" SHORT_SHA1=${SHA1: -8} @@ -45,13 +50,19 @@ resolve_os_version_for_repo() { printf '%s\n' "$os_label" } -# Return the number of repository versions to retain. +# Return the number of repository versions to retain from purge-policy.yaml. +# Uses keep_minimum for BRANCH under ceph.ref when present; otherwise the default. get_repo_versions_to_retain() { - local versions=10 - if [[ "$CEPH_REPO" == *-ci ]]; then - versions=3 - fi - printf '%s\n' "$versions" + local keep_minimum + + keep_minimum=$( + yq -r --arg ref "${BRANCH}" \ + --arg project "${PULP_PROJECT}" \ + '.[$project].ref[$ref].keep_minimum + // .[$project].default.keep_minimum' \ + "${PURGE_POLICY_FILE}" + ) + printf '%s\n' "$keep_minimum" } # Create a Pulp repository if it doesn't exist. diff --git a/scripts/purge-policy.yaml b/scripts/purge-policy.yaml new file mode 100644 index 000000000..ea33ed996 --- /dev/null +++ b/scripts/purge-policy.yaml @@ -0,0 +1,58 @@ +ceph: + default: + days: 14 + keep_minimum: 3 + ref: + reef-nvmeof: + keep_minimum: 2 + squid-nvmeof: + keep_minimum: 2 + squid-nvmeof-8.0: + keep_minimum: 2 + wip-rbd-cgsm-base: + keep_minimum: 2 + hammer: + days: 30 + keep_minimum: 5 + infernalis: + days: 30 + keep_minimum: 5 + jewel: + days: 14 + keep_minimum: 10 + kraken: + days: 14 + keep_minimum: 10 + luminous: + days: 14 + keep_minimum: 10 + mimic: + days: 14 + keep_minimum: 10 + nautilus: + days: 14 + keep_minimum: 10 + octopus: + days: 14 + keep_minimum: 10 + pacific: + days: 14 + keep_minimum: 10 + quincy: + days: 14 + keep_minimum: 10 + reef: + days: 14 + keep_minimum: 10 + squid: + days: 14 + keep_minimum: 10 + tentacle: + days: 14 + keep_minimum: 10 + umbrella: + days: 14 + keep_minimum: 10 + vampire: + days: 14 + keep_minimum: 10 diff --git a/scripts/setup_yq.sh b/scripts/setup_yq.sh new file mode 100755 index 000000000..5cec413b4 --- /dev/null +++ b/scripts/setup_yq.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# vim: ts=4 sw=4 expandtab + +function setup_yq () { + # Check if yq is already installed + if command -v yq >/dev/null 2>&1; then + echo "yq already installed: $(yq --version)" + return 0 + fi + + # Install yq + command -v yq || ( + command -v apt && sudo apt install -y yq + command -v dnf && sudo dnf install -y yq + ) || true + + # Verify yq installation + yq --version +} + +# If the script is executed (as opposed to sourced), run the function now +if [ "$(basename -- "${0#-}")" = "$(basename -- "${BASH_SOURCE[0]}")" ]; then + setup_yq +fi