pulp-cleanup: add nightly Jenkins job for stale Pulp resource cleanup

pulp-cleanup/build/Jenkinsfile

@@ -0,0 +1,68 @@
+import groovy.transform.Field
+
+@Field String pulpServerUrl = "https://pulp.front.sepia.ceph.com"
+
+
+pipeline {
+    agent {
+        label 'small'
+    }
+
+    options {
+        timeout(time: 1, unit: 'HOURS')
+        timestamps()
+    }
+
+    stages {
+        stage('Checkout') {
+            steps {
+                checkout scm
+            }
+        }
+
+        stage('Setup Pulp Client') {
+            steps {
+                script {
+                    withCredentials([
+                        usernamePassword(
+                            credentialsId: 'pulp-admin-auth',
+                            usernameVariable: 'PULP_USERNAME',
+                            passwordVariable: 'PULP_PASSWORD'
+                        )
+                    ]) {
+                        echo "Setting up Pulp Client..."
+                        sh """#!/bin/bash -ex
+                            export PULP_SERVER_URL="${pulpServerUrl}" && ./scripts/setup_pulp.sh
+                        """
+                    }
+                }
+            }
+        }
+
+        stage('Setup yq') {
+            steps {
+                script {
+                    echo "Setting up yq..."
+                    sh """#!/bin/bash -ex
+                        source ./scripts/setup_yq.sh
+                    """
+                }
+            }
+        }
+
+        stage('Execute Pulp Cleanup for Ceph') {
+            when {
+                expression {
+                    params.CEPH
+                }
+            }
+            steps {
+                script {
+                    def project = "ceph"
+                    echo "Starting Pulp Cleanup for project: ${project}"
+                    sh "./pulp-cleanup/build/cleanup.sh ${project}"
+                }
+            }
+        }
+    }
+}

pulp-cleanup/build/cleanup.sh

@@ -0,0 +1,190 @@
+#!/bin/bash
+
+# Purge stale Pulp repositories and distributions per purge-policy.yaml.
+
+set -euo pipefail
+
+readonly PROJECT="$1"
+readonly PURGE_POLICY_FILE="${WORKSPACE}/scripts/purge-policy.yaml"
+
+readonly LABEL="ref"
+readonly PULP_LIST_LIMIT=1000
+readonly PULP_TYPES=(rpm deb)
+readonly PULP_RESOURCES=(distribution repository)
+
+# Default protection time in hours
+readonly PROTECTION_TIME=24
+
+log() {
+    echo "[pulp_cleanup] $*" >&2
+}
+
+get_policy_refs() {
+    local project="$1"
+    local label="$2"
+
+    yq -r ".${project}.${label} | keys[]" "${PURGE_POLICY_FILE}"
+}
+
+read_default_purge_policy() {
+    local project="$1"
+    local -n _days="$2"
+    local policy_line
+
+    policy_line=$(
+        yq -r \
+            ".${project}.default | [(.days // \"\")] | @tsv" \
+            "${PURGE_POLICY_FILE}"
+    )
+    IFS=$'\t' read -r _days <<< "${policy_line}"
+}
+
+fetch_pulp_resource_json() {
+    local type="$1"
+    local resource="$2"
+    local project="$3"
+    local label_select="$4"
+
+    pulp "${type}" "${resource}" list \
+        --limit "${PULP_LIST_LIMIT}" \
+        --ordering '-pulp_created' \
+        --label-select project="${project}","${label_select}" \
+        2>&1 | sed -n '/^\[/,$p'
+}
+
+filter_resource_names_by_age() {
+    local pulp_json="$1"
+    local days="$2"
+
+    if [ -z "${days}" ]; then
+        echo "${pulp_json}" | jq -r '.[].name'
+        return 0
+    fi
+
+    echo "${pulp_json}" | jq -r --arg days "${days}" '
+        .[]
+        | select(
+            (.pulp_created | split(".")[0] + "Z" | fromdateiso8601)
+            < (now - ($days | tonumber) * 86400)
+          )
+        | .name
+    '
+}
+
+list_stale_pulp_resources() {
+    local type="$1"
+    local resource="$2"
+    local project="$3"
+    local label_select="$4"
+    local days="$5"
+    local pulp_json resource_list
+
+    pulp_json=$(
+        fetch_pulp_resource_json \
+            "${type}" "${resource}" "${project}" "${label_select}"
+    )
+    resource_list=$(
+        filter_resource_names_by_age "${pulp_json}" "${days}"
+    )
+    echo "${resource_list}"
+}
+
+destroy_pulp_resources() {
+    local type="$1"
+    local resource="$2"
+    local resource_list="$3"
+    local name
+
+    if [ -z "${resource_list}" ]; then
+        return 0
+    fi
+
+    while IFS= read -r name; do
+        [ -z "${name}" ] && continue
+        log "Destroying ${type} ${resource}: ${name}"
+        if ! pulp "${type}" "${resource}" destroy \
+                --name "${name}"; then
+            log "ERROR: Failed to destroy ${type} ${resource}: ${name}"
+            return 1
+        fi
+    done <<< "${resource_list}"
+}
+
+collect_stale_resources_for_ref() {
+    local project="$1"
+    local label_select="$2"
+    local days="$3"
+    local type resource key
+
+    declare -n _stale_resources="$4"
+
+    for type in "${PULP_TYPES[@]}"; do
+        for resource in "${PULP_RESOURCES[@]}"; do
+            key="${type}_${resource}"
+            _stale_resources[$key]=$(
+                list_stale_pulp_resources \
+                    "${type}" "${resource}" "${project}" \
+                    "${label_select}" "${days}"
+            )
+            log "${key}: ${_stale_resources[$key]:-}"
+        done
+    done
+}
+
+purge_resources() {
+    local project="$1"
+    local label_select="$2"
+    local days="$3"
+    local type resource
+
+    declare -A stale_resources=()
+    log "Purging days=${days:-unset} label_select=${label_select}"
+    collect_stale_resources_for_ref "${project}" "${label_select}" "${days}" \
+        stale_resources
+
+    log "stale_resources: ${stale_resources[@]}"
+
+    for type in "${PULP_TYPES[@]}"; do
+        destroy_pulp_resources "${type}" "distribution" \
+            "${stale_resources[${type}_distribution]:-}"
+    done
+    for type in "${PULP_TYPES[@]}"; do
+        destroy_pulp_resources "${type}" "repository" \
+            "${stale_resources[${type}_repository]:-}"
+    done
+}
+
+run_package_cleanup() {
+    local project="$1"
+    local label="$2"
+    local days
+
+    mapfile -t policy_refs < <(get_policy_refs "${project}" "${label}")
+    _label_select=$(
+        printf "${label}!=%s\n" "${policy_refs[@]}" | paste -sd, -
+    )
+    log "label_select: ${_label_select}"
+
+    read_default_purge_policy "${project}" days
+    log "days: ${days}"
+
+    purge_resources "${project}" "${_label_select}" "${days}"
+}
+
+run_orphan_cleanup() {
+    log "Applying orphan content purge policy"
+    if ! pulp orphan cleanup \
+            --protection-time "${PROTECTION_TIME}"; then
+        log "ERROR: Failed to run orphan cleanup"
+        return 1
+    fi
+    log "Orphan cleanup completed"
+}
+
+log "Cleaning up project: ${PROJECT}"
+
+log "Running package cleanup"
+run_package_cleanup "${PROJECT}" "${LABEL}"
+
+log "Running orphan cleanup"
+run_orphan_cleanup

pulp-cleanup/config/definitions/pulp-cleanup.yaml

@@ -0,0 +1,37 @@
+- job:
+    name: pulp-cleanup-nightly
+    disabled: true
+    display-name: 'Pulp: Cleanup Nightly'
+    project-type: pipeline
+    description: >
+      Nightly parameterized cleanup of stale Teuthology Pulp resources.
+    defaults: global
+
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: pulp-cleanup/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+
+    triggers:
+      - timed: '@midnight'
+
+    parameters:
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: 'Use the Jenkinsfile from this ceph-build branch'
+          default: 'main'
+      - bool:
+          name: CEPH
+          description: >
+            When true, clean up stale Pulp resources for the ceph project
+            per purge-policy.yaml.
+          default: true

scripts/pulp_upload.sh

@@ -25,7 +25,12 @@ source ./scripts/build_utils.sh
 
 export PATH="$HOME/.local/bin:$PATH"
 
+# Pulp project name
 PULP_PROJECT="ceph"
+
+# Set purge policy file
+PURGE_POLICY_FILE="${WORKSPACE}/scripts/purge-policy.yaml"
+
 # Must match the base URL the Pulp client is configured with in setup_pulp.sh
 PULP_SERVER_URL="https://pulp.front.sepia.ceph.com"
 SHORT_SHA1=${SHA1: -8}
@@ -45,13 +50,19 @@ resolve_os_version_for_repo() {
     printf '%s\n' "$os_label"
 }
 
-# Return the number of repository versions to retain.
+# Return the number of repository versions to retain from purge-policy.yaml.
+# Uses keep_minimum for BRANCH under ceph.ref when present; otherwise the default.
 get_repo_versions_to_retain() {
-    local versions=10
-    if [[ "$CEPH_REPO" == *-ci ]]; then
-        versions=3
-    fi
-    printf '%s\n' "$versions"
+    local keep_minimum
+
+    keep_minimum=$(
+        yq -r --arg ref "${BRANCH}" \
+            --arg project "${PULP_PROJECT}" \
+            '.[$project].ref[$ref].keep_minimum
+             // .[$project].default.keep_minimum' \
+            "${PURGE_POLICY_FILE}"
+    )
+    printf '%s\n' "$keep_minimum"
 }
 
 # Create a Pulp repository if it doesn't exist.

scripts/purge-policy.yaml

@@ -0,0 +1,58 @@
+ceph:
+  default:
+    days: 14
+    keep_minimum: 3
+  ref:
+    reef-nvmeof:
+      keep_minimum: 2
+    squid-nvmeof:
+      keep_minimum: 2
+    squid-nvmeof-8.0:
+      keep_minimum: 2
+    wip-rbd-cgsm-base:
+      keep_minimum: 2
+    hammer:
+      days: 30
+      keep_minimum: 5
+    infernalis:
+      days: 30
+      keep_minimum: 5
+    jewel:
+      days: 14
+      keep_minimum: 10
+    kraken:
+      days: 14
+      keep_minimum: 10
+    luminous:
+      days: 14
+      keep_minimum: 10
+    mimic:
+      days: 14
+      keep_minimum: 10
+    nautilus:
+      days: 14
+      keep_minimum: 10
+    octopus:
+      days: 14
+      keep_minimum: 10
+    pacific:
+      days: 14
+      keep_minimum: 10
+    quincy:
+      days: 14
+      keep_minimum: 10
+    reef:
+      days: 14
+      keep_minimum: 10
+    squid:
+      days: 14
+      keep_minimum: 10
+    tentacle:
+      days: 14
+      keep_minimum: 10
+    umbrella:
+      days: 14
+      keep_minimum: 10
+    vampire:
+      days: 14
+      keep_minimum: 10