From 97a86f9ff5a9b33581d894d38e1a7c839eae2b3a Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Tue, 23 Jun 2026 10:08:02 +0000 Subject: [PATCH 01/13] cleanup k8s script files also Signed-off-by: Nagachandan-P --- .../tasks/cleanup_k8s.yml | 28 +++++++++++++++++-- .../oim_container_cleanup/vars/main.yml | 19 +++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml b/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml index 6f0da6b43e..3b6d39633b 100644 --- a/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml +++ b/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml @@ -121,9 +121,10 @@ - name: Display k8s cleanup information ansible.builtin.debug: msg: | - WARNING: This will delete K8s-related directories from NFS shares: + WARNING: This will delete K8s-related directories and files from NFS shares: {% for mount in k8s_storage_mounts %} Storage: {{ mount.name }} ({{ mount.mount_point }}) + Directories: {% for item in k8s_static_dirs_stat.results %} {% if item.stat.exists and item.item.startswith(mount.mount_point) %} - {{ item.item }} ({{ item.item | basename }}) @@ -141,6 +142,10 @@ {% else %} Node IP directories: Skipped (k8s_cleanup_node_ips: false) {% endif %} + Root-level files: + {% for file in k8s_cleanup_files %} + - {{ mount.mount_point }}/{{ file }} + {% endfor %} {% endfor %} CRITICAL WARNING: Deleting NFS shared data will affect ALL nodes! @@ -173,6 +178,17 @@ when: k8s_cleanup_needed | default(false) loop: "{{ k8s_all_cleanup_paths }}" + - name: Delete K8s root-level files + ansible.builtin.file: + path: "{{ item.0 }}/{{ item.1 }}" + state: absent + register: k8s_files_cleanup_result + when: k8s_cleanup_needed | default(false) + loop: "{{ all_k8s_base_paths | product(k8s_cleanup_files) | list }}" + loop_control: + label: "{{ item.0 }}/{{ item.1 }}" + failed_when: false + - name: Display k8s cleanup completion message ansible.builtin.debug: msg: | @@ -182,10 +198,18 @@ {% set mount_deleted = k8s_cleanup_result.results | selectattr('item', 'search', '^' + mount.mount_point) | selectattr('changed') | list %} {% if mount_deleted %} {% for item in mount_deleted %} - -> Deleted: {{ item.item }} + -> Deleted directory: {{ item.item }} {% endfor %} {% else %} -> No directories deleted from this storage {% endif %} + {% set mount_files_deleted = k8s_files_cleanup_result.results | selectattr('item', 'search', '^' + mount.mount_point) | selectattr('changed') | list %} + {% if mount_files_deleted %} + {% for item in mount_files_deleted %} + -> Deleted file: {{ item.item }} + {% endfor %} + {% else %} + -> No files deleted from this storage + {% endif %} {% endfor %} when: k8s_cleanup_needed | default(false) diff --git a/utils/roles/oim_cleanup/oim_container_cleanup/vars/main.yml b/utils/roles/oim_cleanup/oim_container_cleanup/vars/main.yml index 28dd327351..5f3d60ea72 100644 --- a/utils/roles/oim_cleanup/oim_container_cleanup/vars/main.yml +++ b/utils/roles/oim_cleanup/oim_container_cleanup/vars/main.yml @@ -218,13 +218,14 @@ oim_cleanup_note: | - For Slurm configuration backup, use the separate utility: ansible-playbook utils/slurm_config_util.yml --tags config_backup - To skip slurm cleanup, run: ansible-playbook utils/oim_cleanup.yml --skip-tags slurm - 3. The playbook removes K8s-related directories from NFS shares: - - ssh, calico, metallb, helm, packages, telemetry, karavi-observability, csi-driver-powerscale, nfs-client-provisioner + 3. The playbook removes K8s-related directories and files from NFS shares: + - Directories: ssh, calico, metallb, helm, packages, telemetry, karavi-observability, csi-driver-powerscale, nfs-client-provisioner + - Files: control-plane-join-command.sh, generate-control-plane-join.sh, worker-join-command.sh, pulp_webserver.crt - Node IP directories (when k8s_cleanup_node_ips: true) - - Directory list is configurable via k8s_cleanup_directories variable in vars/main.yml - - Supports multi-storage: Cleans directories from all K8s storage mounts configured in omnia_config.yml + - Directory and file lists are configurable via k8s_cleanup_directories and k8s_cleanup_files variables in vars/main.yml + - Supports multi-storage: Cleans directories and files from all K8s storage mounts configured in omnia_config.yml - To skip k8s cleanup, run: ansible-playbook utils/oim_cleanup.yml --skip-tags k8s - - No backup is created for k8s directories (directory deletion only) + - No backup is created for k8s directories and files (deletion only) 4. The omnia_core container is NOT removed by oim_cleanup.yml. - To delete it, log in to the OIM node and run: @@ -248,6 +249,14 @@ k8s_cleanup_directories: - csi-driver-powerscale - nfs-client-provisioner +# List of k8s root-level files to delete from NFS share +# Edit this list to add/remove files as needed +k8s_cleanup_files: + - control-plane-join-command.sh + - generate-control-plane-join.sh + - worker-join-command.sh + - pulp_webserver.crt + # Delete node IP directories (pattern: x.x.x.x) # Set to false to skip node directories k8s_cleanup_node_ips: true From 6e97680e4cd3c21ddca9df214b9961374396ddbb Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Tue, 23 Jun 2026 11:00:21 +0000 Subject: [PATCH 02/13] lint issue fix Signed-off-by: Nagachandan-P --- .../oim_container_cleanup/tasks/cleanup_k8s.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml b/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml index 3b6d39633b..52e324cd76 100644 --- a/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml +++ b/utils/roles/oim_cleanup/oim_container_cleanup/tasks/cleanup_k8s.yml @@ -195,7 +195,10 @@ K8s-related cleanup completed. {% for mount in k8s_storage_mounts %} Storage: {{ mount.name }} ({{ mount.mount_point }}) - {% set mount_deleted = k8s_cleanup_result.results | selectattr('item', 'search', '^' + mount.mount_point) | selectattr('changed') | list %} + {% set mount_deleted = k8s_cleanup_result.results | + selectattr('item', 'search', '^' + mount.mount_point) | + selectattr('changed') | + list %} {% if mount_deleted %} {% for item in mount_deleted %} -> Deleted directory: {{ item.item }} @@ -203,7 +206,10 @@ {% else %} -> No directories deleted from this storage {% endif %} - {% set mount_files_deleted = k8s_files_cleanup_result.results | selectattr('item', 'search', '^' + mount.mount_point) | selectattr('changed') | list %} + {% set mount_files_deleted = k8s_files_cleanup_result.results | + selectattr('item', 'search', '^' + mount.mount_point) | + selectattr('changed') | + list %} {% if mount_files_deleted %} {% for item in mount_files_deleted %} -> Deleted file: {{ item.item }} From 4c064e375c3381eaaa6202b6cb39f0f19e3c4c44 Mon Sep 17 00:00:00 2001 From: Nagachandan-P Date: Wed, 24 Jun 2026 06:28:40 +0000 Subject: [PATCH 03/13] mpi env set for default Signed-off-by: Nagachandan-P --- .../cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 | 4 +--- .../cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 | 5 +---- .../cloud_init/ci-group-slurm_node_aarch64.yaml.j2 | 6 +----- .../templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 | 6 +----- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 index dfdaa37111..5879e0ba3d 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_aarch64.yaml.j2 @@ -281,18 +281,16 @@ {% endif %} # UCX and OpenMPI auto-compilation disabled - # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 used by default + # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 configured by default {% if hostvars['localhost']['ucx_support'] %} - echo "===== UCX Configuration =====" - echo "UCX version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA UCX 1.20.0 (system default)" {% endif %} -{% if hostvars['localhost']['openmpi_support'] %} - echo "===== OpenMPI Configuration =====" - echo "OpenMPI version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA OpenMPI 4.1.9a1 (system default)" - bash /usr/local/bin/setup_doca_mpi_env.sh || echo "DOCA MPI environment setup failed (non-critical)" -{% endif %} {% if ldms_support %} - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 index 0c727b0c01..77109a4828 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-login_compiler_node_x86_64.yaml.j2 @@ -282,19 +282,16 @@ {% endif %} # UCX and OpenMPI auto-compilation disabled - # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 used by default + # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 configured by default {% if hostvars['localhost']['ucx_support'] %} - echo "===== UCX Configuration =====" - echo "UCX version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA UCX 1.20.0 (system default)" {% endif %} -{% if hostvars['localhost']['openmpi_support'] %} - echo "===== OpenMPI Configuration =====" - echo "OpenMPI version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA OpenMPI 4.1.9a1 (system default)" - bash /usr/local/bin/setup_doca_mpi_env.sh || echo "DOCA MPI environment setup failed (non-critical)" - -{% endif %} {% if ldms_support %} - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 index cdce20193e..449cedfa8f 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_aarch64.yaml.j2 @@ -422,9 +422,7 @@ # DOCA and IB configuration - now ready before vendor_data mounts - bash /usr/local/bin/doca-install.sh || true - bash /usr/local/bin/configure-ib-network.sh -{% if hostvars['localhost']['openmpi_support'] %} - bash /usr/local/bin/setup_doca_mpi_env.sh || echo "DOCA MPI environment setup failed (non-critical)" -{% endif %} {# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined and cloud_init_groups_dict[functional_group_name].runcmd is not none %} @@ -513,17 +511,15 @@ - mount -av {% endif %} # UCX and OpenMPI auto-compilation disabled - # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 used by default + # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 configured by default {% if hostvars['localhost']['ucx_support'] %} - echo "===== UCX Configuration =====" - echo "UCX version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA UCX 1.20.0 (system default)" {% endif %} -{% if hostvars['localhost']['openmpi_support'] %} - echo "===== OpenMPI Configuration =====" - echo "OpenMPI version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA OpenMPI 4.1.9a1 (system default)" -{% endif %} {% if ldms_support %} - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log diff --git a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 index ee33e0ff6f..ecf388cdeb 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/ci-group-slurm_node_x86_64.yaml.j2 @@ -428,9 +428,7 @@ # DOCA and IB configuration - now ready before vendor_data mounts - bash /usr/local/bin/doca-install.sh || echo "DOCA install failed (non-critical)" - bash /usr/local/bin/configure-ib-network.sh || echo "IB network configuration failed (non-critical)" -{% if hostvars['localhost']['openmpi_support'] %} - bash /usr/local/bin/setup_doca_mpi_env.sh || echo "DOCA MPI environment setup failed (non-critical)" -{% endif %} {# Mount-specific runcmd entries - moved after DOCA to ensure RDMA is available #} {%- if cloud_init_groups_dict[functional_group_name].runcmd is defined and cloud_init_groups_dict[functional_group_name].runcmd is not none %} @@ -515,17 +513,15 @@ - mount -av {% endif %} # UCX and OpenMPI auto-compilation disabled - # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 used by default + # DOCA UCX 1.20.0 and OpenMPI 4.1.9a1 configured by default {% if hostvars['localhost']['ucx_support'] %} - echo "===== UCX Configuration =====" - echo "UCX version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA UCX 1.20.0 (system default)" {% endif %} -{% if hostvars['localhost']['openmpi_support'] %} - echo "===== OpenMPI Configuration =====" - echo "OpenMPI version specified in software_config.json (available for manual compilation)" - echo "Default stack - DOCA OpenMPI 4.1.9a1 (system default)" -{% endif %} {% if ldms_support %} - echo " Starting LDMS setup " | tee -a /var/log/ldms-cloudinit.log From eb9798e897e596ec65d815991803f199c083efa1 Mon Sep 17 00:00:00 2001 From: Kratika Patidar Date: Fri, 26 Jun 2026 14:46:40 +0530 Subject: [PATCH 04/13] Merge pull request #4786 from Kratika-P/pub/q2_upgrade Telemetry component version updates --- .../catalog/tests/test_generate_catalog_id.py | 139 ++++++++++++++---- examples/catalog/catalog_rhel.json | 16 +- .../catalog_rhel_with_nfs_provisioner.json | 16 +- examples/catalog/catalog_rhel_x86_64.json | 16 +- .../x86_64/rhel/10.0/service_k8s_v1.35.1.json | 8 +- .../templates/coredhcp/coredhcp.yaml.j2 | 2 +- .../telemetry/kafka/kafka.kafka.yaml.j2 | 10 +- .../kafka/kafka.kafka_bridge.yaml.j2 | 3 +- .../kafka/kafka.kafkapump_user.yaml.j2 | 2 +- .../telemetry/kafka/kafka.topic.yaml.j2 | 2 +- .../templates/telemetry/telemetry.sh.j2 | 11 ++ .../vector/vector-ome-kafkauser.yaml.j2 | 2 +- provision/roles/telemetry/vars/main.yml | 6 +- .../files/migrate_strimzi_crds.sh | 117 +++++++++++++++ .../tasks/apply_victoria_crs.yml | 2 +- .../tasks/execute_telemetry_sh.yml | 16 ++ 16 files changed, 295 insertions(+), 73 deletions(-) create mode 100644 upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh diff --git a/build_stream/core/catalog/tests/test_generate_catalog_id.py b/build_stream/core/catalog/tests/test_generate_catalog_id.py index 00807b3e41..c71f15c7e3 100644 --- a/build_stream/core/catalog/tests/test_generate_catalog_id.py +++ b/build_stream/core/catalog/tests/test_generate_catalog_id.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Unit tests for the _generate_human_readable_id function.""" +# pylint: disable=wrong-import-position + import unittest import sys from pathlib import Path @@ -21,61 +24,137 @@ from generate_catalog import _generate_human_readable_id + class TestGenerateHumanReadableId(unittest.TestCase): + """Test cases for _generate_human_readable_id function.""" + def setUp(self): + """Set up test fixtures before each test method.""" self.used_ids = set() def test_basic_names(self): - # Should remain unchanged - self.assertEqual(_generate_human_readable_id("apptainer", "rpm", None, self.used_ids), "apptainer") - self.assertEqual(_generate_human_readable_id("device-mapper-multipath", "rpm", None, self.used_ids), "device-mapper-multipath") + """Test that basic package names remain unchanged.""" + self.assertEqual( + _generate_human_readable_id("apptainer", "rpm", None, self.used_ids), + "apptainer" + ) + self.assertEqual( + _generate_human_readable_id( + "device-mapper-multipath", "rpm", None, self.used_ids + ), + "device-mapper-multipath" + ) def test_version_in_name_exact(self): - # Should strip exact version suffix - self.assertEqual(_generate_human_readable_id("external-snapshotter-v8.4.0", "git", "v8.4.0", self.used_ids), "external-snapshotter") - + """Test stripping exact version suffix.""" + self.assertEqual( + _generate_human_readable_id( + "external-snapshotter-v8.4.0", "git", "v8.4.0", self.used_ids + ), + "external-snapshotter" + ) + def test_version_in_name_v_prefixed(self): - # Should strip if 'v' prefix is in name but not in version - self.assertEqual(_generate_human_readable_id("app-v1.0.0", "rpm", "1.0.0", self.used_ids), "app") + """Test stripping version with 'v' prefix in name.""" + self.assertEqual( + _generate_human_readable_id("app-v1.0.0", "rpm", "1.0.0", self.used_ids), + "app" + ) def test_version_in_name_dots_replaced(self): - # Should strip if dots are replaced by hyphens - self.assertEqual(_generate_human_readable_id("helm-charts-2-16-0", "git", "2.16.0", self.used_ids), "helm-charts") + """Test stripping version when dots are replaced by hyphens.""" + self.assertEqual( + _generate_human_readable_id("helm-charts-2-16-0", "git", "2.16.0", self.used_ids), + "helm-charts" + ) def test_pip_module_format(self): - # PyMySQL==1.1.2 -> PyMySQL - self.assertEqual(_generate_human_readable_id("PyMySQL==1.1.2", "pip_module", None, self.used_ids), "PyMySQL") + """Test pip module format with == version separator.""" + self.assertEqual( + _generate_human_readable_id("PyMySQL==1.1.2", "pip_module", None, self.used_ids), + "PyMySQL" + ) def test_regex_fallback_no_version(self): - # Regex should strip the version even without explicit pkg_version - self.assertEqual(_generate_human_readable_id("calico-v3.31.4", "manifest", None, self.used_ids), "calico") - self.assertEqual(_generate_human_readable_id("cert-manager-v1-10-0", "tarball", None, self.used_ids), "cert-manager") - self.assertEqual(_generate_human_readable_id("helm-v3-20-1-amd64", "tarball", None, self.used_ids), "helm-amd64") - self.assertEqual(_generate_human_readable_id("helm-v3-20-1-chart", "tarball", None, self.used_ids), "helm-chart") - self.assertEqual(_generate_human_readable_id("helm-v3-20-1-anything-suffixed", "tarball", None, self.used_ids), "helm-anything-suffixed") - self.assertEqual(_generate_human_readable_id("metallb-native-v0-15-3", "manifest", None, self.used_ids), "metallb-native") - self.assertEqual(_generate_human_readable_id("strimzi-kafka-operator-helm-3-chart-0-48-0", "tarball", None, self.used_ids), "strimzi-kafka-operator-helm-3-chart") - self.assertEqual(_generate_human_readable_id("victoria-metrics-operator-0-59-3", "tarball", None, self.used_ids), "victoria-metrics-operator") - self.assertEqual(_generate_human_readable_id("python3-PyMySQL-1.1.2", "rpm", None, self.used_ids), "python3-PyMySQL") - self.assertEqual(_generate_human_readable_id("nfs-subdir-external-provisioner-4-0-18", "tarball", None, self.used_ids), "nfs-subdir-external-provisioner") - + """Test regex version stripping without explicit pkg_version.""" + self.assertEqual( + _generate_human_readable_id("calico-v3.31.4", "manifest", None, self.used_ids), + "calico" + ) + self.assertEqual( + _generate_human_readable_id( + "cert-manager-v1-10-0", "tarball", None, self.used_ids + ), + "cert-manager" + ) + self.assertEqual( + _generate_human_readable_id("helm-v3-20-1-amd64", "tarball", None, self.used_ids), + "helm-amd64" + ) + self.assertEqual( + _generate_human_readable_id("helm-v3-20-1-chart", "tarball", None, self.used_ids), + "helm-chart" + ) + self.assertEqual( + _generate_human_readable_id( + "helm-v3-20-1-anything-suffixed", "tarball", None, self.used_ids + ), + "helm-anything-suffixed" + ) + self.assertEqual( + _generate_human_readable_id( + "metallb-native-v0-15-3", "manifest", None, self.used_ids + ), + "metallb-native" + ) + self.assertEqual( + _generate_human_readable_id( + "strimzi-kafka-operator-helm-3-chart-1-0-1", "tarball", None, + self.used_ids + ), + "strimzi-kafka-operator-helm-3-chart" + ) + self.assertEqual( + _generate_human_readable_id( + "victoria-metrics-operator-0-59-3", "tarball", None, self.used_ids + ), + "victoria-metrics-operator" + ) + self.assertEqual( + _generate_human_readable_id("python3-PyMySQL-1.1.2", "rpm", None, + self.used_ids), + "python3-PyMySQL" + ) + self.assertEqual( + _generate_human_readable_id( + "nfs-subdir-external-provisioner-4-0-18", "tarball", None, + self.used_ids + ), + "nfs-subdir-external-provisioner" + ) + def test_docker_image_without_tag_in_name(self): - # Docker images usually don't have the tag in the name field, just the image path - self.assertEqual(_generate_human_readable_id("docker.io/library/python", "image", "3.12-slim", self.used_ids), "docker.io/library/python") + """Test docker images where tag is not in the name.""" + self.assertEqual( + _generate_human_readable_id( + "docker.io/library/python", "image", "3.12-slim", self.used_ids + ), + "docker.io/library/python" + ) def test_collision_handling(self): - # First call gets the base name + """Test collision handling with _1, _2 suffixes.""" id1 = _generate_human_readable_id("calico", "rpm", None, self.used_ids) self.assertEqual(id1, "calico") - + # Second call with the same base name gets _1 id2 = _generate_human_readable_id("calico-v1.0.0", "tarball", None, self.used_ids) self.assertEqual(id2, "calico_1") - + # Third call gets _2 id3 = _generate_human_readable_id("calico-v2.0.0", "manifest", None, self.used_ids) self.assertEqual(id3, "calico_2") - + self.assertIn("calico", self.used_ids) self.assertIn("calico_1", self.used_ids) self.assertIn("calico_2", self.used_ids) diff --git a/examples/catalog/catalog_rhel.json b/examples/catalog/catalog_rhel.json index fa265ce93b..fe8ac00cd1 100644 --- a/examples/catalog/catalog_rhel.json +++ b/examples/catalog/catalog_rhel.json @@ -1862,8 +1862,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0-kafka-4.1.0", - "Version": "0.48.0-kafka-4.1.0" + "Tag": "1.0.1-kafka-4.2.0", + "Version": "1.0.1-kafka-4.2.0" }, "quay.io/strimzi/kafka-bridge": { "Name": "quay.io/strimzi/kafka-bridge", @@ -1877,8 +1877,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.33.1", - "Version": "0.33.1" + "Tag": "1.0.0", + "Version": "1.0.0" }, "quay.io/strimzi/operator": { "Name": "quay.io/strimzi/operator", @@ -1892,8 +1892,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0", - "Version": "0.48.0" + "Tag": "1.0.1", + "Version": "1.0.1" }, "registry.k8s.io/coredns/coredns": { "Name": "registry.k8s.io/coredns/coredns", @@ -2184,7 +2184,7 @@ ] }, "strimzi-kafka-operator-helm-3-chart": { - "Name": "strimzi-kafka-operator-helm-3-chart-0.48.0", + "Name": "strimzi-kafka-operator-helm-3-chart-1.0.1", "SupportedOS": [ { "Name": "RHEL", @@ -2198,7 +2198,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" + "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/1.0.1/strimzi-kafka-operator-helm-3-chart-1.0.1.tgz" } ] }, diff --git a/examples/catalog/catalog_rhel_with_nfs_provisioner.json b/examples/catalog/catalog_rhel_with_nfs_provisioner.json index 511d5d0a6c..d11135023a 100644 --- a/examples/catalog/catalog_rhel_with_nfs_provisioner.json +++ b/examples/catalog/catalog_rhel_with_nfs_provisioner.json @@ -1735,8 +1735,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0-kafka-4.1.0", - "Version": "0.48.0-kafka-4.1.0" + "Tag": "1.0.1-kafka-4.2.0", + "Version": "1.0.1-kafka-4.2.0" }, "quay.io/strimzi/kafka-bridge": { "Name": "quay.io/strimzi/kafka-bridge", @@ -1750,8 +1750,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.33.1", - "Version": "0.33.1" + "Tag": "1.0.0", + "Version": "1.0.0" }, "quay.io/strimzi/operator": { "Name": "quay.io/strimzi/operator", @@ -1765,8 +1765,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0", - "Version": "0.48.0" + "Tag": "1.0.1", + "Version": "1.0.1" }, "registry.k8s.io/coredns/coredns": { "Name": "registry.k8s.io/coredns/coredns", @@ -2057,7 +2057,7 @@ ] }, "strimzi-kafka-operator-helm-3-chart": { - "Name": "strimzi-kafka-operator-helm-3-chart-0.48.0", + "Name": "strimzi-kafka-operator-helm-3-chart-1.0.1", "SupportedOS": [ { "Name": "RHEL", @@ -2071,7 +2071,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" + "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/1.0.1/strimzi-kafka-operator-helm-3-chart-1.0.1.tgz" } ] }, diff --git a/examples/catalog/catalog_rhel_x86_64.json b/examples/catalog/catalog_rhel_x86_64.json index a882551226..d0c97cbf25 100644 --- a/examples/catalog/catalog_rhel_x86_64.json +++ b/examples/catalog/catalog_rhel_x86_64.json @@ -1752,8 +1752,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0-kafka-4.1.0", - "Version": "0.48.0-kafka-4.1.0" + "Tag": "1.0.1-kafka-4.2.0", + "Version": "1.0.1-kafka-4.2.0" }, "quay.io/strimzi/kafka-bridge": { "Name": "quay.io/strimzi/kafka-bridge", @@ -1767,8 +1767,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.33.1", - "Version": "0.33.1" + "Tag": "1.0.0", + "Version": "1.0.0" }, "quay.io/strimzi/operator": { "Name": "quay.io/strimzi/operator", @@ -1782,8 +1782,8 @@ "x86_64" ], "Type": "image", - "Tag": "0.48.0", - "Version": "0.48.0" + "Tag": "1.0.1", + "Version": "1.0.1" }, "registry.k8s.io/coredns/coredns": { "Name": "registry.k8s.io/coredns/coredns", @@ -2039,7 +2039,7 @@ ] }, "strimzi-kafka-operator-helm-3-chart": { - "Name": "strimzi-kafka-operator-helm-3-chart-0.48.0", + "Name": "strimzi-kafka-operator-helm-3-chart-1.0.1", "SupportedOS": [ { "Name": "RHEL", @@ -2053,7 +2053,7 @@ "Sources": [ { "Architecture": "x86_64", - "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" + "Uri": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/1.0.1/strimzi-kafka-operator-helm-3-chart-1.0.1.tgz" } ] }, diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index 8f21b2bf51..8c18422fcc 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -33,8 +33,8 @@ { "package": "cffi==1.17.1", "type": "pip_module" }, { "package": "prometheus_client==0.20.0", "type": "pip_module" }, { "package": "kubernetes==33.1.0", "type": "pip_module" }, - { "package": "quay.io/strimzi/operator", "tag": "0.48.0", "type": "image" }, - { "package": "quay.io/strimzi/kafka", "tag": "0.48.0-kafka-4.1.0", "type": "image" }, + { "package": "quay.io/strimzi/operator", "tag": "1.0.1", "type": "image" }, + { "package": "quay.io/strimzi/kafka", "tag": "1.0.1-kafka-4.2.0", "type": "image" }, { "package": "docker.io/dellhpcomniaaisolution/ubuntu-ldms", "tag": "1.1", "type": "image" }, { "package": "quay.io/dell/container-storage-modules/csm-metrics-powerscale", "tag": "v1.12.0", "type": "image" }, { "package": "ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector", "tag": "0.150.1", "type": "image" }, @@ -46,8 +46,8 @@ { "package": "quay.io/jetstack/cert-manager-webhook", "tag": "v1.10.0", "type": "image" }, { "package": "quay.io/jetstack/cert-manager-acmesolver", "tag": "v1.10.0", "type": "image" }, { "package": "cert-manager-v1.10.0", "type": "tarball", "url": "https://charts.jetstack.io/charts/cert-manager-v1.10.0.tgz" }, - { "package": "strimzi-kafka-operator-helm-3-chart-0.48.0", "type": "tarball", "url": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" }, - { "package": "quay.io/strimzi/kafka-bridge", "tag": "0.33.1", "type": "image" }, + { "package": "strimzi-kafka-operator-helm-3-chart-1.0.1", "type": "tarball", "url": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/1.0.1/strimzi-kafka-operator-helm-3-chart-1.0.1.tgz" }, + { "package": "quay.io/strimzi/kafka-bridge", "tag": "1.0.0", "type": "image" }, { "package": "docker.io/victoriametrics/operator", "tag": "v0.68.3", "type": "image" }, { "package": "docker.io/victoriametrics/operator", "tag": "config-reloader-v0.68.3", "type": "image" }, { "package": "victoria-metrics-operator-0.59.3", "type": "tarball", "url": "https://github.com/VictoriaMetrics/helm-charts/releases/download/victoria-metrics-operator-0.59.3/victoria-metrics-operator-0.59.3.tgz" }, diff --git a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 index edfcb8c583..1cf09db3f9 100644 --- a/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 +++ b/prepare_oim/roles/deploy_containers/openchami/templates/coredhcp/coredhcp.yaml.j2 @@ -41,7 +41,7 @@ server4: # ------------------------------------------------------------------- # Multi-subnet configuration (requires coresmd v0.6.x+) # To enable multi-subnet DHCP: - # 1. Pull the new coresmd image: podman pull ghcr.io/openchami/coresmd:v0.6.x + # 1. Pull the new coresmd image: podman pull ghcr.io/openchami/coresmd:v0.6.3 # 2. Comment out the single-subnet coresmd and bootloop lines above # 3. Uncomment the multi-subnet coresmd and bootloop blocks below # 4. Replace the new coresmd image version in files: /etc/containers/systemd/coresmd-coredhcp.container /etc/containers/systemd/coresmd-coredns.container with the old version diff --git a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka.yaml.j2 index 929af037c4..a9a0f6196f 100644 --- a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka.yaml.j2 @@ -1,4 +1,4 @@ -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: name: controller @@ -19,7 +19,7 @@ spec: deleteClaim: false --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaNodePool metadata: name: broker @@ -40,7 +40,7 @@ spec: deleteClaim: false --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: Kafka metadata: name: kafka @@ -50,8 +50,8 @@ metadata: strimzi.io/kraft: enabled spec: kafka: - version: 4.1.0 - metadataVersion: 4.1-IV0 + version: 4.2.0 + metadataVersion: 4.2-IV0 listeners: - name: internal port: 9092 diff --git a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka_bridge.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka_bridge.yaml.j2 index 35a0862cf3..b4fdb5689c 100644 --- a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka_bridge.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafka_bridge.yaml.j2 @@ -1,12 +1,11 @@ --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaBridge metadata: name: bridge namespace: telemetry spec: bootstrapServers: kafka-kafka-bootstrap:9093 - enableMetrics: true http: port: 8080 # Enable TLS for Kafka connection diff --git a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafkapump_user.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafkapump_user.yaml.j2 index 413a7fe72d..df1b9f9bae 100644 --- a/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafkapump_user.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kafka/kafka.kafkapump_user.yaml.j2 @@ -13,7 +13,7 @@ # limitations under the License. --- -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaUser metadata: name: kafkapump diff --git a/provision/roles/telemetry/templates/telemetry/kafka/kafka.topic.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kafka/kafka.topic.yaml.j2 index 9ae180ecfd..974712e78f 100644 --- a/provision/roles/telemetry/templates/telemetry/kafka/kafka.topic.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kafka/kafka.topic.yaml.j2 @@ -1,4 +1,4 @@ -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaTopic metadata: name: {{ topic_name }} diff --git a/provision/roles/telemetry/templates/telemetry/telemetry.sh.j2 b/provision/roles/telemetry/templates/telemetry/telemetry.sh.j2 index c711863f48..dcc5affb41 100644 --- a/provision/roles/telemetry/templates/telemetry/telemetry.sh.j2 +++ b/provision/roles/telemetry/templates/telemetry/telemetry.sh.j2 @@ -19,6 +19,17 @@ else helm -n telemetry install strimzi-cluster-operator "${DEPLOY_DIR}/{{ strimzi_kafka_pkg }}.tar.gz" fi +# Helm 3 does NOT update CRDs on 'helm upgrade'. It also may not apply CRDs +# immediately on 'helm install' in some configurations. Explicitly apply CRDs +# from the chart tarball so that API versions are registered before +# kubectl apply -k attempts to create Kafka resources. +# This step is idempotent and safe for both fresh installs and upgrades. +echo " Applying Strimzi CRDs from chart..." +_STRIMZI_CRD_TMP=$(mktemp -d) +tar -xzf "${DEPLOY_DIR}/{{ strimzi_kafka_pkg }}.tar.gz" -C "$_STRIMZI_CRD_TMP" +kubectl apply -f "$_STRIMZI_CRD_TMP/strimzi-kafka-operator/crds/" --server-side --force-conflicts +rm -rf "$_STRIMZI_CRD_TMP" + # Wait for Strimzi operator to be ready echo " Waiting for Strimzi operator deployment..." kubectl wait --for=condition=available --timeout=300s deployment/strimzi-cluster-operator -n telemetry diff --git a/provision/roles/telemetry/templates/telemetry/vector/vector-ome-kafkauser.yaml.j2 b/provision/roles/telemetry/templates/telemetry/vector/vector-ome-kafkauser.yaml.j2 index 2cad636058..28e69319f3 100644 --- a/provision/roles/telemetry/templates/telemetry/vector/vector-ome-kafkauser.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/vector/vector-ome-kafkauser.yaml.j2 @@ -9,7 +9,7 @@ # OME is an external producer with a different security domain, so it gets a # dedicated, least-privilege KafkaUser. -apiVersion: kafka.strimzi.io/v1beta2 +apiVersion: kafka.strimzi.io/v1 kind: KafkaUser metadata: name: {{ vector.ome.kafka_user }} diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 4f1f29f9fa..1337efb093 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -116,9 +116,9 @@ kafka: lb_service_name: "kafka-loadbalancer" container_port1: 9093 # Kafka images from service_k8s_v.json - operator_image: "{{ telemetry_images['strimzi/operator'] | default('quay.io/strimzi/operator:0.48.0') }}" - kafka_image: "{{ telemetry_images['strimzi/kafka'] | default('quay.io/strimzi/kafka:0.48.0-kafka-4.1.0') }}" - bridge_image: "{{ telemetry_images['strimzi/kafka-bridge'] | default('quay.io/strimzi/kafka-bridge:0.33.1') }}" + operator_image: "{{ telemetry_images['strimzi/operator'] | default('quay.io/strimzi/operator:1.0.1') }}" + kafka_image: "{{ telemetry_images['strimzi/kafka'] | default('quay.io/strimzi/kafka:1.0.1-kafka-4.2.0') }}" + bridge_image: "{{ telemetry_images['strimzi/kafka-bridge'] | default('quay.io/strimzi/kafka-bridge:1.0.0') }}" container_port2: 9093 image: "apache/kafka:4.1.0" cluster_id: "kafka-cluster-id" diff --git a/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh b/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh new file mode 100644 index 0000000000..9916018d8c --- /dev/null +++ b/upgrade/roles/upgrade_telemetry/files/migrate_strimzi_crds.sh @@ -0,0 +1,117 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#!/bin/bash +# migrate_strimzi_crds.sh — Strimzi CRD major version migration +# +# Handles the upgrade from Strimzi 0.x (v1beta2) to 1.x (v1-only). +# Strimzi 1.0.x completely dropped the v1beta2 API. Kubernetes +# cannot remove a served version when objects stored in that version +# still exist in etcd. This script: +# +# 1. Detects whether migration is needed +# 2. Temporarily re-enables v1beta2 on CRDs so stuck CRs are readable +# 3. Deletes existing Kafka CRs (they will be recreated by telemetry.sh) +# 4. Deletes old PVCs (new cluster ID makes old data incompatible) +# 5. Removes CRDs (handles stuck cleanup finalizers) +# +# telemetry.sh then recreates CRDs + CRs from the new chart. +# This script is fully idempotent — it is a no-op when CRDs are +# already healthy, absent, or running v1 without issues. +# +# Usage: bash migrate_strimzi_crds.sh +# Exit codes: 0 = success or no migration needed + +set -euo pipefail + +NS="${1:-telemetry}" + +# ── Phase 1: Detect ───────────────────────────────────────────── +needs_migration=false + +# Check if any Strimzi CRD still lists v1beta2 in storedVersions +for crd in $(kubectl get crd -o name 2>/dev/null | grep -E '\.kafka\.strimzi\.io|\.core\.strimzi\.io'); do + if kubectl get "$crd" -o jsonpath='{.status.storedVersions}' 2>/dev/null | grep -q 'v1beta2'; then + echo "[MIGRATE] $crd has v1beta2 in storedVersions" + needs_migration=true + break + fi +done + +# Check if CRs are stuck (v1-only CRDs but objects stored as v1beta2) +if [ "$needs_migration" = "false" ] && kubectl get crd kafkas.kafka.strimzi.io >/dev/null 2>&1; then + if kubectl get kafka -n "$NS" 2>&1 | grep -q 'convert CR from an invalid group/version'; then + echo "[MIGRATE] CRs stuck — conversion error detected" + needs_migration=true + fi +fi + +if [ "$needs_migration" = "false" ]; then + echo "[MIGRATE] No Strimzi CRD migration needed." + exit 0 +fi + +echo "[MIGRATE] Starting Strimzi CRD migration (v1beta2 → v1)..." + +# ── Phase 2: Make stuck CRs readable ──────────────────────────── +STRIMZI_CRDS=$(kubectl get crd -o name 2>/dev/null \ + | grep -E '\.kafka\.strimzi\.io|\.core\.strimzi\.io' \ + | sed 's|customresourcedefinition.apiextensions.k8s.io/||') + +if [ -n "$STRIMZI_CRDS" ]; then + echo "[MIGRATE] Temporarily adding v1beta2 to CRDs..." + for crd in $STRIMZI_CRDS; do + kubectl get crd "$crd" -o json 2>/dev/null \ + | jq '.spec.versions += [(.spec.versions[0] | .name = "v1beta2" | .served = true | .storage = false)]' \ + | kubectl apply -f - --server-side --force-conflicts >/dev/null 2>&1 || true + done +fi + +# ── Phase 3: Delete existing CRs ──────────────────────────────── +echo "[MIGRATE] Deleting existing Kafka CRs..." +for kind in kafka kafkanodepool kafkabridge kafkatopic kafkauser strimzipodset; do + for item in $(kubectl get "$kind" -n "$NS" -o name 2>/dev/null); do + kubectl patch "$item" -n "$NS" --type=merge \ + -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true + kubectl delete "$item" -n "$NS" --wait=false 2>/dev/null || true + done +done +sleep 5 + +# ── Phase 4: Delete old Kafka PVCs ─────────────────────────────── +echo "[MIGRATE] Deleting old Kafka PVCs (new cluster ID makes old data incompatible)..." +kubectl delete pvc -n "$NS" -l strimzi.io/cluster=kafka --wait=false 2>/dev/null || true + +# ── Phase 5: Delete cluster-id secret (operator will regenerate) ─ +kubectl delete secret kafka-cluster-id -n "$NS" 2>/dev/null || true + +# ── Phase 6: Delete CRDs ──────────────────────────────────────── +if [ -n "$STRIMZI_CRDS" ]; then + echo "[MIGRATE] Deleting Strimzi CRDs..." + kubectl delete crd $STRIMZI_CRDS --wait=false --timeout=30s 2>&1 || true + sleep 5 + # Remove cleanup finalizers from any CRDs stuck in Terminating + for crd in $(kubectl get crd -o name 2>/dev/null | grep -E '\.strimzi\.io'); do + kubectl patch "$crd" --type=merge \ + -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true + done + # Wait for CRDs to fully disappear + for i in $(seq 1 24); do + remaining=$(kubectl get crd -o name 2>/dev/null | grep -cE '\.strimzi\.io' || echo 0) + [ "$remaining" -eq 0 ] 2>/dev/null && break + sleep 5 + done +fi + +echo "[MIGRATE] Strimzi CRD migration complete. telemetry.sh will recreate CRDs and CRs." diff --git a/upgrade/roles/upgrade_telemetry/tasks/apply_victoria_crs.yml b/upgrade/roles/upgrade_telemetry/tasks/apply_victoria_crs.yml index 1e8b7aa3f0..cc9a8fc1c6 100644 --- a/upgrade/roles/upgrade_telemetry/tasks/apply_victoria_crs.yml +++ b/upgrade/roles/upgrade_telemetry/tasks/apply_victoria_crs.yml @@ -150,7 +150,7 @@ - name: Reclaim preserved IPs from conflicting services when: - preserved_vminsert_ip | default('') | length > 0 or preserved_vmselect_ip | default('') | length > 0 - - vminsert_lb_ip.stdout | trim | length == 0 or vmselect_lb_ip.stdout | trim | length == 0 + - (vminsert_lb_ip.stdout | default('') | trim | length == 0) or (vmselect_lb_ip.stdout | default('') | trim | length == 0) block: - name: Stage IP conflict detection script ansible.builtin.template: diff --git a/upgrade/roles/upgrade_telemetry/tasks/execute_telemetry_sh.yml b/upgrade/roles/upgrade_telemetry/tasks/execute_telemetry_sh.yml index 1dac883990..1f0715b8ac 100644 --- a/upgrade/roles/upgrade_telemetry/tasks/execute_telemetry_sh.yml +++ b/upgrade/roles/upgrade_telemetry/tasks/execute_telemetry_sh.yml @@ -67,6 +67,22 @@ ansible.builtin.debug: msg: "{{ pods_before_upgrade.stdout_lines }}" + # ── Pre-CRD migration: Strimzi major version upgrade (0.x → 1.x) ── + # See files/migrate_strimzi_crds.sh for full details. + # The script is idempotent — no-op when CRDs are already healthy or absent. + - name: Run Strimzi CRD migration if needed (v1beta2 → v1) + ansible.builtin.script: + cmd: migrate_strimzi_crds.sh {{ telemetry_namespace }} + delegate_to: "{{ kube_vip }}" + connection: ssh + register: strimzi_migration_result + changed_when: "'starting strimzi crd migration' in strimzi_migration_result.stdout | lower" + failed_when: strimzi_migration_result.rc != 0 + + - name: Display Strimzi migration result + ansible.builtin.debug: + msg: "{{ strimzi_migration_result.stdout_lines }}" + # ── Execute telemetry.sh ── - name: Execute telemetry.sh on kube_vip ansible.builtin.command: From fb60485b1963249b608d0ef1b85fcd1e703bb565 Mon Sep 17 00:00:00 2001 From: "balajikumaran.cs" Date: Fri, 26 Jun 2026 16:26:46 +0530 Subject: [PATCH 05/13] Fix: Add --limit 500 to regctl repo ls to prevent 26th build failure (#4796) * input_validation changes for the powerscale telemetry * Update en_us_validation_msg.py Signed-off-by: balajikumaran.cs * Update en_us_validation_msg.py Signed-off-by: balajikumaran.cs * Update en_us_validation_msg.py Signed-off-by: balajikumaran.cs * s3_cmd and victoria_metrics changes * Update cleanup_openchami.yml Signed-off-by: balajikumaran.cs * lint fix * Update cleanup_openchami.yml Signed-off-by: balajikumaran.cs * Fix: Add --limit 500 to regctl repo ls to prevent 26th build failure Root Cause: - regctl repo ls defaults to 100 repos max (Docker registry v2 pagination) - After 25 builds (100 compute images), registry verification fails - Base image becomes 101st repo alphabetically and is truncated from results Resolution: - Added --limit 500 flag to all regctl repo ls commands in image_creation tasks - Applies to both x86_64 and aarch64 base/compute image verification Files Changed: - build_image_x86_64/roles/image_creation/tasks/build_base_image.yml (line 69) - build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml (line 97) - build_image_aarch64/roles/image_creation/tasks/build_base_image.yml (line 79) - build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml (line 105) Tested: regctl repo ls --limit 500 returns all 101 repos including base image --------- Signed-off-by: balajikumaran.cs --- .../roles/image_creation/tasks/build_base_image.yml | 2 +- .../roles/image_creation/tasks/build_compute_image.yml | 2 +- .../roles/image_creation/tasks/build_base_image.yml | 2 +- .../roles/image_creation/tasks/build_compute_image.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_image_aarch64/roles/image_creation/tasks/build_base_image.yml b/build_image_aarch64/roles/image_creation/tasks/build_base_image.yml index 42cdc9fef0..82704d76a2 100644 --- a/build_image_aarch64/roles/image_creation/tasks/build_base_image.yml +++ b/build_image_aarch64/roles/image_creation/tasks/build_base_image.yml @@ -76,7 +76,7 @@ - name: Verify the aarch64 base osimage in registry ansible.builtin.command: - cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000" + cmd: "/usr/local/bin/regctl repo ls --limit 500 {{ oim_node_name }}.{{ domain_name }}:5000" delegate_to: "{{ aarch64_build_host }}" connection: ssh changed_when: false diff --git a/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml b/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml index 0ca77db3df..38eff4480e 100644 --- a/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml +++ b/build_image_aarch64/roles/image_creation/tasks/build_compute_image.yml @@ -102,7 +102,7 @@ - name: Verify aarch64 compute osimages in registry ansible.builtin.command: - cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000" + cmd: "/usr/local/bin/regctl repo ls --limit 500 {{ oim_node_name }}.{{ domain_name }}:5000" delegate_to: "{{ aarch64_build_host }}" connection: ssh changed_when: false diff --git a/build_image_x86_64/roles/image_creation/tasks/build_base_image.yml b/build_image_x86_64/roles/image_creation/tasks/build_base_image.yml index 7e4bfa445e..7966db4e92 100644 --- a/build_image_x86_64/roles/image_creation/tasks/build_base_image.yml +++ b/build_image_x86_64/roles/image_creation/tasks/build_base_image.yml @@ -66,7 +66,7 @@ - name: Verify the x86_64 base osimage in registry ansible.builtin.command: - cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000" + cmd: "/usr/local/bin/regctl repo ls --limit 500 {{ oim_node_name }}.{{ domain_name }}:5000" changed_when: false register: verify_base_osimage diff --git a/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml b/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml index 84f6a2b063..eb00755486 100644 --- a/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml +++ b/build_image_x86_64/roles/image_creation/tasks/build_compute_image.yml @@ -94,7 +94,7 @@ - name: Verify x86_64 compute osimages in registry ansible.builtin.command: - cmd: "/usr/local/bin/regctl repo ls {{ oim_node_name }}.{{ domain_name }}:5000" + cmd: "/usr/local/bin/regctl repo ls --limit 500 {{ oim_node_name }}.{{ domain_name }}:5000" changed_when: false register: verify_compute_osimages From ccf49ad071708b7b63f267ce6baf6159f287404a Mon Sep 17 00:00:00 2001 From: Abhishek S A Date: Mon, 29 Jun 2026 14:37:13 +0530 Subject: [PATCH 06/13] Update restore_quadlets_and_configs.yml (#4800) Signed-off-by: Abhishek S A --- .../tasks/restore_quadlets_and_configs.yml | 98 +++++++++++++++++-- 1 file changed, 88 insertions(+), 10 deletions(-) diff --git a/rollback/roles/rollback_openchami/tasks/restore_quadlets_and_configs.yml b/rollback/roles/rollback_openchami/tasks/restore_quadlets_and_configs.yml index 092c7b576e..139546acb5 100644 --- a/rollback/roles/rollback_openchami/tasks/restore_quadlets_and_configs.yml +++ b/rollback/roles/rollback_openchami/tasks/restore_quadlets_and_configs.yml @@ -22,6 +22,8 @@ # 3. Restore v2.1 openchami.target from backup (references coresmd.service) # 4. Restore v2.1 configs (coredhcp.yaml, Corefile, configs_vars.yaml) # 5. Restore v2.1 RPMs (openchami + ochami CLI) from backup +# NOTE: backup may contain both v2.1 AND v2.2 RPMs — filter needed +# 5a. Re-remove v2.2 quadlets and re-restore configs after RPM install # 6. Reload systemd daemon # # Backup structure (created by omnia.sh --upgrade): @@ -197,9 +199,41 @@ connection: ssh # ── 5. Restore v2.1 RPMs from backup ──────────────────────────────── - # The backup openchami_data/ may contain the v2.1 RPMs that were - # installed before the upgrade. If they exist, reinstall them to - # restore the v2.1 quadlet file definitions and CLI version. + # IMPORTANT: The backup openchami_data/ contains BOTH v2.1 AND v2.2 + # RPMs (the upgrade downloads v2.2 RPMs before creating the backup). + # Installing all RPMs causes v2.2 to win (processed last → upgrades + # the just-downgraded v2.1), re-creating v2.2 quadlet files and + # overwriting configs with RPM defaults. + # + # Fix: Read openchami_backup_manifest.yml which records the exact + # source (v2.1) and target (v2.2) RPM filenames. Install ONLY the + # source RPMs. + - name: Read openchami backup manifest for source RPM versions + ansible.builtin.slurp: + src: "{{ rollback_oim_host_backup_dir }}/openchami_backup_manifest.yml" + register: rollback_ochami_manifest_raw + delegate_to: oim + delegate_facts: true + connection: ssh + failed_when: false + + - name: Parse openchami backup manifest + ansible.builtin.set_fact: + rollback_ochami_manifest: >- + {{ (rollback_ochami_manifest_raw.content | b64decode | from_yaml) + if rollback_ochami_manifest_raw is not failed + and rollback_ochami_manifest_raw.content is defined + else {} }} + + - name: Set v2.1 RPM filenames from manifest source_rpms + ansible.builtin.set_fact: + rollback_v21_rpm_names: >- + {{ [ + rollback_ochami_manifest.source_rpms.openchami | default(''), + rollback_ochami_manifest.source_rpms.ochami_cli | default('') + ] | select | list }} + when: rollback_ochami_manifest.source_rpms is defined + - name: Find backed-up openchami RPMs ansible.builtin.find: paths: "{{ rollback_oim_host_data }}" @@ -211,27 +245,71 @@ connection: ssh failed_when: false - - name: Display found RPMs in backup + - name: Filter RPMs to v2.1 only (match manifest source_rpms) + ansible.builtin.set_fact: + rollback_v21_rpms: >- + {%- set result = [] -%} + {%- for rpm in rollback_rpm_files.files | default([]) -%} + {%- if rpm.path | basename in (rollback_v21_rpm_names | default([])) -%} + {%- set _ = result.append(rpm.path) -%} + {%- endif -%} + {%- endfor -%} + {{ result }} + + - name: Display RPMs selected for rollback ansible.builtin.debug: verbosity: 1 msg: >- - Found {{ rollback_rpm_files.files | default([]) | length }} RPM(s) in backup: - {{ rollback_rpm_files.files | default([]) | map(attribute='path') | map('basename') | list }} + All RPMs in backup: {{ rollback_rpm_files.files | default([]) | map(attribute='path') | map('basename') | list }}. + Manifest source_rpms: {{ rollback_v21_rpm_names | default([]) }}. + Selected for install: {{ rollback_v21_rpms | default([]) | map('basename') | list }} - name: Reinstall v2.1 RPMs from backup (downgrade) ansible.builtin.dnf: - name: "{{ item.path }}" + name: "{{ item }}" state: present disable_gpg_check: true allow_downgrade: true - loop: "{{ rollback_rpm_files.files | default([]) }}" + loop: "{{ rollback_v21_rpms | default([]) }}" loop_control: - label: "{{ item.path | basename }}" + label: "{{ item | basename }}" delegate_to: oim delegate_facts: true connection: ssh failed_when: false + # ── 5a. Re-remove v2.2 quadlets and re-restore configs after RPM ─── + # Defense in depth: RPM post-install scripts may still create files. + # Re-apply backup state to guarantee v2.1 quadlets and configs. + - name: Re-remove v2.2-only quadlets after RPM reinstall + ansible.builtin.file: + path: "{{ systemd_quadlet_dir }}/{{ item }}" + state: absent + loop: "{{ rollback_v22_only_quadlets }}" + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Re-restore v2.1 quadlets after RPM reinstall + ansible.builtin.shell: | + set -o pipefail + cp -a {{ rollback_oim_host_quadlets }}/*.container {{ systemd_quadlet_dir }}/ + cp -a {{ rollback_oim_host_quadlets }}/*.network {{ systemd_quadlet_dir }}/ 2>/dev/null || true + changed_when: true + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Re-restore /etc/openchami after RPM reinstall + ansible.builtin.shell: | + set -o pipefail + cp -a {{ rollback_oim_host_etc_openchami }}/. {{ openchami_config_dir | regex_replace('/configs$', '') }}/ + when: rollback_etc_openchami_backup_stat.stat.exists | default(false) + changed_when: true + delegate_to: oim + delegate_facts: true + connection: ssh + # ── 6. Reload systemd daemon ──────────────────────────────────────── # Apply all quadlet and target changes before starting services. - name: Reload systemd daemon to apply restored quadlets and target @@ -251,4 +329,4 @@ - "openchami.target: restored from backup (references coresmd.service)" - "/etc/openchami: {{ 'restored from backup' if rollback_etc_openchami_backup_stat.stat.exists | default(false) else 'backup NOT found' }}" - "configs_vars.yaml: {{ 'restored' if rollback_configvars_stat.stat.exists | default(false) else 'not in backup' }}" - - "RPMs restored: {{ rollback_rpm_files.files | default([]) | length }}" + - "RPMs restored (v2.1 only): {{ rollback_v21_rpms | default([]) | map('basename') | list }}" From 2d0c1ee935afe54bf15bb3d0761c6b2cea8bfa4d Mon Sep 17 00:00:00 2001 From: pullan1 Date: Mon, 29 Jun 2026 21:43:04 +0530 Subject: [PATCH 07/13] pulp upgrade and rollback Signed-off-by: pullan1 --- .../library/modules/pulp_fs_orphan_cleanup.py | 373 ++++++++++++++ .../modules/pulp_pgsql_backup_restore.py | 459 ++++++++++++++++++ rollback/playbooks/rollback_oim.yml | 54 ++- .../tasks/normalize_permissions.yml | 62 ++- .../roles/rollback_openchami/vars/main.yml | 31 +- .../rollback_pulp/tasks/cleanup_pulp_data.yml | 99 ++++ rollback/roles/rollback_pulp/tasks/main.yml | 88 ++++ .../tasks/post_rollback_health_check.yml | 91 ++++ .../tasks/pre_rollback_checks.yml | 134 +++++ .../rollback_pulp/tasks/resolve_admin_ip.yml | 95 ++++ .../rollback_pulp/tasks/restore_pulp_data.yml | 41 ++ .../tasks/rollback_pulp_container.yml | 131 +++++ .../rollback_pulp/tasks/rollback_status.yml | 52 ++ rollback/roles/rollback_pulp/vars/main.yml | 120 +++++ upgrade/playbooks/upgrade_oim.yml | 65 ++- .../upgrade_pulp/tasks/backup_pulp_data.yml | 37 ++ .../tasks/cleanup_after_upgrade.yml | 57 +++ upgrade/roles/upgrade_pulp/tasks/main.yml | 87 ++++ .../tasks/post_upgrade_health_check.yml | 75 +++ .../tasks/pre_upgrade_health_check.yml | 100 ++++ .../upgrade_pulp/tasks/resolve_admin_ip.yml | 95 ++++ .../tasks/upgrade_pulp_container.yml | 108 +++++ upgrade/roles/upgrade_pulp/vars/main.yml | 105 ++++ upgrade/upgrade.yml | 4 +- 24 files changed, 2526 insertions(+), 37 deletions(-) create mode 100644 common/library/modules/pulp_fs_orphan_cleanup.py create mode 100644 common/library/modules/pulp_pgsql_backup_restore.py create mode 100644 rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml create mode 100644 rollback/roles/rollback_pulp/tasks/main.yml create mode 100644 rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml create mode 100644 rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml create mode 100644 rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml create mode 100644 rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml create mode 100644 rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml create mode 100644 rollback/roles/rollback_pulp/tasks/rollback_status.yml create mode 100644 rollback/roles/rollback_pulp/vars/main.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/main.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml create mode 100644 upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml create mode 100644 upgrade/roles/upgrade_pulp/vars/main.yml diff --git a/common/library/modules/pulp_fs_orphan_cleanup.py b/common/library/modules/pulp_fs_orphan_cleanup.py new file mode 100644 index 0000000000..1fd4b1cca7 --- /dev/null +++ b/common/library/modules/pulp_fs_orphan_cleanup.py @@ -0,0 +1,373 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=import-error,no-name-in-module +#!/usr/bin/python + +""" +Remove filesystem orphan artifacts from Pulp storage. + +After a PostgreSQL restore (rollback), artifact files synced during the +upgrade exist on disk but are not referenced by the rolled-back database. +Pulp's built-in orphan cleanup only checks DB records and cannot detect +these filesystem-level orphans. + +This module: + 1. Reads Pulp credentials from cli.toml (no password in Ansible args) + 2. Queries the Pulp REST API to get all known artifact file paths + 3. Scans the artifact directory on disk + 4. Removes files present on disk but absent from the database + 5. Cleans up empty directories + +Authentication: + Credentials are read from the Pulp CLI config at + /root/.config/pulp/cli.toml (shared NFS mount available in omnia_core). + The raw password is never stored as an instance attribute. +""" + +import base64 +import http.client +import json +import os +import ssl + +from ansible.module_utils.basic import AnsibleModule + + +PULP_CLI_CONFIG_PATH = "/root/.config/pulp/cli.toml" + + +# ============================================================================= +# Pulp CLI config reading (same pattern as pulp_repo_name_migration.py) +# ============================================================================= + +def _read_toml_config(): + """Read and parse the Pulp CLI TOML config file. + + Returns the parsed dict, or None on failure. + Separated from credential extraction so that credential handling + does not share the same scope as file I/O. + """ + try: + import toml as toml_mod + except ImportError: + try: + import tomllib as toml_mod # Python 3.11+ + except ImportError: + try: + import tomli as toml_mod + except ImportError: + return None + + if not os.path.isfile(PULP_CLI_CONFIG_PATH): + return None + + try: + if hasattr(toml_mod, "loads"): + with open(PULP_CLI_CONFIG_PATH, "r", encoding="utf-8") as fh: + return toml_mod.loads(fh.read()) + else: + # tomllib requires binary mode + with open(PULP_CLI_CONFIG_PATH, "rb") as fb: + return toml_mod.load(fb) + except Exception: + return None + + +def _build_auth_header(cli_section): + """Build a Basic Authorization header from a config section. + + Reads username and password directly from the dict and produces + the encoded header. The raw password is never stored beyond this + helper's local scope. + """ + credential = ( + (cli_section.get("username") or "admin") + + ":" + + (cli_section.get("password") or "") + ).encode("utf-8") + header = "Basic " + base64.b64encode(credential).decode("utf-8") + # Overwrite the byte string that held the combined credential. + credential = b"" # noqa: F841 + return header + + +def _load_pulp_config(): + """Load Pulp base URL and auth header from cli.toml. + + Returns (base_url, auth_header) or (None, None) on failure. + """ + cfg = _read_toml_config() + if cfg is None: + return None, None + + try: + cli_section = cfg.get("cli", {}) + base_url = cli_section.get("base_url", "https://localhost") + + # Enforce HTTPS + if base_url.startswith("http://"): + base_url = "https://" + base_url[len("http://"):] + + auth_header = _build_auth_header(cli_section) + return base_url, auth_header + except Exception: + return None, None + + +# ============================================================================= +# Pulp REST API helpers +# ============================================================================= + +def _pulp_api_get(base_url, endpoint, auth_header): + """Make a GET request to the Pulp API using http.client. + + Uses json.JSONDecoder instead of json.loads to avoid Checkmarx + vulnerability flags. + """ + # Parse base_url to extract host and port + url_no_scheme = base_url.split("://", 1)[-1] + host_port = url_no_scheme.split("/", 1)[0] + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + conn = http.client.HTTPSConnection(host_port, context=ctx, timeout=30) + try: + conn.request("GET", endpoint, headers={ + "Authorization": auth_header, + "Accept": "application/json", + }) + resp = conn.getresponse() + data = resp.read().decode("utf-8") + if resp.status != 200: + return None + decoder = json.JSONDecoder() + parsed, _ = decoder.raw_decode(data.strip()) + return parsed + finally: + conn.close() + + +def _pulp_api_post(base_url, endpoint, auth_header, body_dict): + """Make a POST request to the Pulp API.""" + url_no_scheme = base_url.split("://", 1)[-1] + host_port = url_no_scheme.split("/", 1)[0] + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + body = json.dumps(body_dict).encode("utf-8") + conn = http.client.HTTPSConnection(host_port, context=ctx, timeout=30) + try: + conn.request("POST", endpoint, body=body, headers={ + "Authorization": auth_header, + "Accept": "application/json", + "Content-Type": "application/json", + }) + resp = conn.getresponse() + data = resp.read().decode("utf-8") + decoder = json.JSONDecoder() + parsed, _ = decoder.raw_decode(data.strip()) + return resp.status, parsed + finally: + conn.close() + + +def _collect_db_artifacts(base_url, auth_header): + """Collect all artifact file paths from the Pulp database via REST API.""" + artifacts = set() + offset = 0 + limit = 100 + + # Get total count + result = _pulp_api_get( + base_url, + f"/pulp/api/v3/artifacts/?limit=1&offset=0", + auth_header + ) + if result is None: + return None + total = result.get("count", 0) + + while offset < total: + result = _pulp_api_get( + base_url, + f"/pulp/api/v3/artifacts/?limit={limit}&offset={offset}&fields=file", + auth_header + ) + if result is None: + return None + for artifact in result.get("results", []): + file_path = artifact.get("file", "") + if file_path: + artifacts.add(file_path.lstrip("/")) + offset += limit + + return artifacts + + +def _scan_disk_artifacts(media_dir): + """Scan the artifact directory on disk and return relative paths.""" + artifact_dir = os.path.join(media_dir, "artifact") + if not os.path.isdir(artifact_dir): + return set() + + disk_files = set() + for root, _dirs, files in os.walk(artifact_dir): + for fname in files: + full_path = os.path.join(root, fname) + rel_path = os.path.relpath(full_path, media_dir) + disk_files.add(rel_path) + return disk_files + + +def _remove_empty_dirs(base_dir): + """Remove empty directories bottom-up.""" + if not os.path.isdir(base_dir): + return + for root, dirs, _files in os.walk(base_dir, topdown=False): + for dirname in dirs: + dirpath = os.path.join(root, dirname) + try: + if not os.listdir(dirpath): + os.rmdir(dirpath) + except OSError: + pass + + +# ============================================================================= +# Ansible module +# ============================================================================= + +def run_module(): + """Remove filesystem orphan artifacts from Pulp storage.""" + module_args = dict( + media_dir=dict(type="str", required=True), + trigger_db_orphan_cleanup=dict(type="bool", required=False, default=True), + ) + + result = dict( + changed=False, + db_artifact_count=0, + disk_artifact_count=0, + orphan_count=0, + removed_count=0, + freed_bytes=0, + freed_mb=0, + db_orphan_cleanup_status="", + messages=[], + ) + + module = AnsibleModule(argument_spec=module_args, supports_check_mode=True) + + media_dir = module.params["media_dir"] + trigger_db_cleanup = module.params["trigger_db_orphan_cleanup"] + + if not os.path.isdir(media_dir): + module.fail_json(msg=f"Media directory not found: {media_dir}", **result) + + # --- Load credentials from cli.toml --- + base_url, auth_header = _load_pulp_config() + if base_url is None or auth_header is None: + module.fail_json( + msg=f"Failed to load Pulp config from {PULP_CLI_CONFIG_PATH}", + **result + ) + + # --- Trigger database-level orphan cleanup (optional) --- + if trigger_db_cleanup: + try: + status, resp = _pulp_api_post( + base_url, + "/pulp/api/v3/orphans/cleanup/", + auth_header, + {"orphan_protection_time": 0} + ) + if status == 202: + task_href = resp.get("task", "N/A") + result["db_orphan_cleanup_status"] = f"triggered (task: {task_href})" + else: + result["db_orphan_cleanup_status"] = f"failed (HTTP {status})" + except Exception as exc: + result["db_orphan_cleanup_status"] = f"error: {exc}" + + # --- Collect DB artifacts --- + db_artifacts = _collect_db_artifacts(base_url, auth_header) + if db_artifacts is None: + module.fail_json(msg="Failed to query Pulp API for artifacts", **result) + + result["db_artifact_count"] = len(db_artifacts) + + # --- Scan disk artifacts --- + disk_artifacts = _scan_disk_artifacts(media_dir) + result["disk_artifact_count"] = len(disk_artifacts) + + # --- Find orphans --- + orphans = disk_artifacts - db_artifacts + result["orphan_count"] = len(orphans) + + if not orphans: + result["messages"].append("No filesystem orphans found") + module.exit_json(**result) + + result["messages"].append( + f"Found {len(orphans)} filesystem orphans " + f"(disk: {len(disk_artifacts)}, DB: {len(db_artifacts)})" + ) + + # --- Check mode --- + if module.check_mode: + result["messages"].append("Check mode: no files removed") + module.exit_json(**result) + + # --- Remove orphans --- + removed = 0 + freed = 0 + for rel_path in orphans: + full_path = os.path.join(media_dir, rel_path) + if not os.path.isfile(full_path): + continue + try: + size = os.path.getsize(full_path) + os.remove(full_path) + freed += size + removed += 1 + except OSError: + pass + + # --- Clean up empty directories --- + artifact_dir = os.path.join(media_dir, "artifact") + _remove_empty_dirs(artifact_dir) + + result["changed"] = removed > 0 + result["removed_count"] = removed + result["freed_bytes"] = freed + result["freed_mb"] = freed // (1024 * 1024) + result["messages"].append( + f"Removed {removed} orphan files, freed {result['freed_mb']} MB" + ) + + module.exit_json(**result) + + +def main(): + """Main entry point.""" + run_module() + + +if __name__ == "__main__": + main() diff --git a/common/library/modules/pulp_pgsql_backup_restore.py b/common/library/modules/pulp_pgsql_backup_restore.py new file mode 100644 index 0000000000..b35aeccdc0 --- /dev/null +++ b/common/library/modules/pulp_pgsql_backup_restore.py @@ -0,0 +1,459 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=import-error,no-name-in-module +#!/usr/bin/python + +""" +Backup and restore Pulp PostgreSQL data for upgrade/rollback. + +Handles the PostgreSQL version incompatibility between Pulp versions: + - Pulp 3.80 uses PostgreSQL 12/13 + - Pulp 3.113 uses PostgreSQL 16 + +Backup (action=backup): + - Validates source PostgreSQL data exists + - Checks PG_VERSION to confirm pre-upgrade state (PG 12 or 13) + - Skips if a valid backup already exists + - Copies data with ownership and SELinux context preserved + - Verifies backup integrity + +Restore (action=restore): + The backup may contain: + - data/ with PG 12/13: restore directly + - data/ with PG 16 + data_old.* with PG 12/13: restore data_old as data +""" + +import os +import glob +import shutil +import subprocess + +from ansible.module_utils.basic import AnsibleModule + + +COMPATIBLE_PG_VERSIONS = ("12", "13") + + +def read_pg_version(data_dir): + """Read PG_VERSION file from a PostgreSQL data directory.""" + version_file = os.path.join(data_dir, "PG_VERSION") + if not os.path.isfile(version_file): + return None + try: + with open(version_file, "r", encoding="utf-8") as fh: + return fh.read().strip() + except (OSError, IOError): + return None + + +def find_data_old_dir(base_path): + """Find a data_old or data_old.* directory inside base_path. + + PostgreSQL upgrade creates either: + - data_old (no suffix) - simple upgrade + - data_old.TIMESTAMP - timestamped backup + """ + # First check for exact 'data_old' directory (most common) + data_old_exact = os.path.join(base_path, "data_old") + if os.path.isdir(data_old_exact): + return data_old_exact + + # Then check for data_old.* pattern (timestamped) + pattern = os.path.join(base_path, "data_old.*") + matches = glob.glob(pattern) + for match in matches: + if os.path.isdir(match): + return match + return None + + +def get_dir_owner(path): + """Get UID and GID of a directory.""" + try: + stat = os.stat(path) + return stat.st_uid, stat.st_gid + except OSError: + return None, None + + +def fix_ownership(path, uid, gid): + """Recursively set ownership on a directory tree. + + PostgreSQL inside the Pulp container runs as UID 26 (postgres). + shutil.copytree does not preserve ownership, so we must fix it + after the copy. + """ + try: + for root, dirs, files in os.walk(path): + os.chown(root, uid, gid) + for name in files: + os.chown(os.path.join(root, name), uid, gid) + except OSError as exc: + raise OSError(f"Failed to chown {path} to {uid}:{gid}: {exc}") from exc + + +def fix_selinux_context(path): + """Apply container SELinux context. Non-fatal on failure.""" + try: + subprocess.run( + ["chcon", "-R", "system_u:object_r:container_file_t:s0", path], + check=False, capture_output=True, timeout=120 + ) + except (subprocess.SubprocessError, OSError): + pass + + +# ========================================================================= +# Backup logic +# ========================================================================= + +def run_backup(module, params, result): + """Backup Pulp PostgreSQL data before upgrade. + + Only backs up the PG 12/13 compatible data directory, not the entire + pgsql folder. This avoids backing up unnecessary files like empty + version folders, upgrade scripts, etc. + + Args: + module: AnsibleModule instance. + params: Dict with 'src_path', 'dest_path', 'backup_dir', + and 'compatible_versions'. + result: Mutable result dict. + """ + src_path = params["src_path"] + dest_path = params["dest_path"] + backup_dir = params["backup_dir"] + compat = tuple(params["compatible_versions"]) + + # --- Validate source exists --- + if not os.path.isdir(src_path): + result["messages"].append( + f"SKIP: Source PostgreSQL data not found at {src_path}" + ) + result["skipped"] = True + module.exit_json(**result) + + # --- Find compatible PG data to backup --- + # First check data/, then data_old (if upgrade already happened) + src_data_dir = os.path.join(src_path, "data") + pg_ver = read_pg_version(src_data_dir) + result["source_pg_version"] = pg_ver or "unknown" + + if pg_ver in compat: + # data/ has compatible PG version, backup it directly + backup_source = src_data_dir + result["messages"].append( + f"Source data/ contains PG {pg_ver}, will backup" + ) + else: + # data/ is upgraded, check for data_old with compatible version + data_old_dir = find_data_old_dir(src_path) + if data_old_dir: + old_ver = read_pg_version(data_old_dir) + if old_ver in compat: + backup_source = data_old_dir + result["messages"].append( + f"Source data/ is PG {pg_ver or 'unknown'}, " + f"using data_old (PG {old_ver}) for backup" + ) + else: + module.fail_json( + msg=( + f"Source data/ is PG {pg_ver or 'unknown'} and " + f"data_old is PG {old_ver or 'unknown'}. " + f"Cannot find PG {'/'.join(compat)} data to backup." + ), + **result + ) + else: + module.fail_json( + msg=( + f"Source data/ is PG {pg_ver or 'unknown'} (already upgraded) " + f"and no data_old directory found. " + f"Cannot backup without PG {'/'.join(compat)} data." + ), + **result + ) + + # --- Skip if backup already exists with valid PG 12/13 data --- + backup_data_dir = os.path.join(dest_path, "data") + if os.path.isdir(backup_data_dir): + backup_ver = read_pg_version(backup_data_dir) + if backup_ver in compat: + result["messages"].append( + f"SKIP: Valid PG {backup_ver} backup already exists at {dest_path}" + ) + result["backup_pg_version"] = backup_ver + result["skipped"] = True + module.exit_json(**result) + + result["messages"].append( + f"WARNING: Existing backup has PG {backup_ver or 'unknown'}, recreating" + ) + if not module.check_mode: + try: + shutil.rmtree(dest_path) + except (OSError, IOError) as exc: + module.fail_json( + msg=f"Failed to remove stale backup at {dest_path}: {exc}", + **result + ) + + # --- Check mode --- + if module.check_mode: + result["messages"].append("Check mode: no changes made") + module.exit_json(**result) + + # --- PostgreSQL requires UID 26 (postgres user) --- + postgres_uid, postgres_gid = 26, 26 + + # --- Create backup (only the data directory) --- + try: + os.makedirs(dest_path, exist_ok=True) + # Backup only the data directory, not the entire pgsql folder + shutil.copytree(backup_source, backup_data_dir, symlinks=True) + except (OSError, IOError, shutil.Error) as exc: + module.fail_json( + msg=f"Backup failed: {exc}", + **result + ) + + # --- Fix ownership (postgres UID:GID = 26:26) --- + try: + fix_ownership(dest_path, postgres_uid, postgres_gid) + result["messages"].append( + f"Set backup ownership to {postgres_uid}:{postgres_gid} (postgres)" + ) + except OSError as exc: + module.fail_json(msg=str(exc), **result) + + # --- Fix SELinux context on backup directory --- + fix_selinux_context(backup_dir) + + # --- Verify --- + if not os.path.isdir(backup_data_dir): + module.fail_json( + msg=f"Backup verification failed - {backup_data_dir} not found", + **result + ) + + final_ver = read_pg_version(backup_data_dir) + if final_ver not in compat: + module.fail_json( + msg=( + f"Backup verification failed - backed up PG {final_ver or 'unknown'}, " + f"expected {'/'.join(compat)}" + ), + **result + ) + + result["backup_pg_version"] = final_ver + result["changed"] = True + result["messages"].append( + f"SUCCESS: Backed up PG {final_ver} data to {backup_data_dir}" + ) + module.exit_json(**result) + + +# ========================================================================= +# Restore logic +# ========================================================================= + +def run_restore(module, params, result): + """Restore Pulp PostgreSQL data for rollback. + + Restores the PG 12/13 data from backup to the destination pgsql directory. + The backup should contain only the data/ directory with compatible PG version. + + Args: + module: AnsibleModule instance. + params: Dict with 'backup_path', 'dest_path', + and 'compatible_versions'. + result: Mutable result dict. + """ + backup_path = params["backup_path"] + dest_path = params["dest_path"] + compat = tuple(params["compatible_versions"]) + + # --- Validate backup exists --- + if not os.path.isdir(backup_path): + module.fail_json( + msg=f"Backup not found at {backup_path}", + **result + ) + + # --- PostgreSQL requires UID 26 (postgres user) --- + postgres_uid, postgres_gid = 26, 26 + + # --- Find compatible PG data in backup --- + backup_data_dir = os.path.join(backup_path, "data") + backup_pg_ver = read_pg_version(backup_data_dir) + result["backup_pg_version"] = backup_pg_ver or "unknown" + + if backup_pg_ver in compat: + # Backup data/ has compatible PG version + restore_source = backup_data_dir + result["restore_mode"] = "direct" + result["messages"].append( + f"Backup contains PG {backup_pg_ver} data, restoring directly" + ) + else: + # Check for data_old in backup (legacy backup format) + data_old_dir = find_data_old_dir(backup_path) + if data_old_dir: + old_pg_ver = read_pg_version(data_old_dir) + if old_pg_ver in compat: + restore_source = data_old_dir + result["restore_mode"] = "data_old" + result["messages"].append( + f"Using backup data_old (PG {old_pg_ver}) for restore" + ) + else: + module.fail_json( + msg=( + f"Backup data/ is PG {backup_pg_ver or 'unknown'} and " + f"data_old is PG {old_pg_ver or 'unknown'}. " + f"Cannot restore without PG {'/'.join(compat)} data." + ), + **result + ) + else: + module.fail_json( + msg=( + f"Backup data/ is PG {backup_pg_ver or 'unknown'} " + f"and no data_old found. " + f"Cannot restore without PG {'/'.join(compat)} data." + ), + **result + ) + + # --- Check mode: report what would happen --- + if module.check_mode: + result["messages"].append("Check mode: no changes made") + module.exit_json(**result) + + # --- Remove current destination pgsql directory --- + if os.path.exists(dest_path): + try: + shutil.rmtree(dest_path) + result["messages"].append(f"Removed existing data at {dest_path}") + except (OSError, IOError) as exc: + module.fail_json( + msg=f"Failed to remove {dest_path}: {exc}", + **result + ) + + # --- Restore: create pgsql/ with only data/ inside --- + try: + os.makedirs(dest_path, exist_ok=True) + dest_data_dir = os.path.join(dest_path, "data") + shutil.copytree(restore_source, dest_data_dir, symlinks=True) + except (OSError, IOError, shutil.Error) as exc: + module.fail_json( + msg=f"Restore failed: {exc}", + **result + ) + + # --- Fix ownership (postgres UID:GID = 26:26) --- + try: + fix_ownership(dest_path, postgres_uid, postgres_gid) + result["messages"].append( + f"Set ownership to {postgres_uid}:{postgres_gid} (postgres)" + ) + except OSError as exc: + module.fail_json(msg=str(exc), **result) + + # --- Fix SELinux context --- + fix_selinux_context(dest_path) + + # --- Verify --- + restored_data_dir = os.path.join(dest_path, "data") + restored_ver = read_pg_version(restored_data_dir) + result["restored_pg_version"] = restored_ver or "unknown" + + if restored_ver not in compat: + module.fail_json( + msg=( + f"Restored data is PG {restored_ver or 'unknown'}, " + f"expected {'/'.join(compat)}" + ), + **result + ) + + result["changed"] = True + result["messages"].append( + f"Restored PG {restored_ver} data to {dest_path}" + ) + module.exit_json(**result) + + +# ========================================================================= +# Module entry point +# ========================================================================= + +def run_module(): + """Ansible module entry point for Pulp PostgreSQL backup/restore.""" + module_args = dict( + action=dict( + type="str", required=True, + choices=["backup", "restore"] + ), + # Common parameters + dest_path=dict(type="str", required=True), + compatible_versions=dict( + type="list", elements="str", required=False, + default=list(COMPATIBLE_PG_VERSIONS) + ), + # Backup-specific parameters + src_path=dict(type="str", required=False, default=""), + backup_dir=dict(type="str", required=False, default=""), + # Restore-specific parameters + backup_path=dict(type="str", required=False, default=""), + ) + + result = dict( + changed=False, + skipped=False, + restore_mode="", + source_pg_version="", + backup_pg_version="", + restored_pg_version="", + messages=[], + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=True, + required_if=[ + ("action", "backup", ["src_path", "dest_path", "backup_dir"]), + ("action", "restore", ["backup_path", "dest_path"]), + ], + ) + + action = module.params["action"] + + if action == "backup": + run_backup(module, module.params, result) + elif action == "restore": + run_restore(module, module.params, result) + + +def main(): + """Main entry point.""" + run_module() + + +if __name__ == "__main__": + main() diff --git a/rollback/playbooks/rollback_oim.yml b/rollback/playbooks/rollback_oim.yml index 05eb34d68f..febab0ac47 100644 --- a/rollback/playbooks/rollback_oim.yml +++ b/rollback/playbooks/rollback_oim.yml @@ -13,7 +13,7 @@ # limitations under the License. --- -- name: Rollback OIM (includes OpenCHAMI) +- name: Rollback OIM (includes Pulp and OpenCHAMI) hosts: localhost connection: local gather_facts: true @@ -59,7 +59,53 @@ ansible.builtin.set_fact: rollback_backup_dir: "{{ rollback_manifest.backup_dir | default('/opt/omnia/backups/upgrade/version_2.1.0.0') }}" - # ── Phase 1: OpenCHAMI Rollback ───────────────────────────────────── + # ── Phase 1: Pulp Rollback ──────────────────────────────────────────── + # The rollback_pulp role handles the full lifecycle: + # resolve_admin_ip → pre_rollback_checks → rollback_pulp_container + # → post_rollback_health_check + # On failure, the role sets pulp_rollback_failed and raises fail. + # On skip (not deployed / already at v2.1), the role completes normally. + - name: "Phase 1 — Pulp Rollback" + block: + - name: Rollback Pulp container + ansible.builtin.include_role: + name: "{{ playbook_dir }}/../roles/rollback_pulp" + + - name: Display Pulp rollback outcome + ansible.builtin.debug: + msg: >- + Pulp Phase 1 result — + deployed={{ pulp_deployed | default(false) }}, + failed={{ pulp_rollback_failed | default(false) }} + + rescue: + - name: Re-read rollback_manifest.yml after Pulp failure + ansible.builtin.slurp: + src: "{{ rollback_manifest_path }}" + register: pulp_rescue_raw_rollback_manifest + + - name: Parse current manifest state + ansible.builtin.set_fact: + pulp_rescue_rollback_manifest: "{{ pulp_rescue_raw_rollback_manifest.content | b64decode | from_yaml }}" + + - name: Pulp rollback failed — mark component as failed + ansible.builtin.copy: + content: >- + {{ pulp_rescue_rollback_manifest | combine({ + 'component_status': pulp_rescue_rollback_manifest.component_status | combine({ + component_name: 'failed' + }) + }) | to_nice_yaml }} + dest: "{{ rollback_manifest_path }}" + mode: '0644' + + - name: Fail with Pulp rollback error + ansible.builtin.fail: + msg: >- + Pulp rollback failed. Check container logs: podman logs pulp + Backup directory: {{ rollback_backup_dir | default('N/A') }} + + # ── Phase 2: OpenCHAMI Rollback ───────────────────────────────────── # The rollback_openchami role handles the full lifecycle: # resolve_backup_dir → pre_rollback_checks → stop_current_containers # → restore_quadlets_and_configs → start_postgres_only @@ -67,7 +113,7 @@ # → reload_cloud_init_data → post_rollback_health_check # On failure, the role sets openchami_rollback_failed and raises fail. # On skip (already at v2.1), the role completes normally. - - name: "Phase 1 — OpenCHAMI Rollback" + - name: "Phase 2 — OpenCHAMI Rollback" block: - name: Rollback OpenCHAMI containers and services ansible.builtin.include_role: @@ -76,7 +122,7 @@ - name: Display OpenCHAMI rollback outcome ansible.builtin.debug: msg: >- - OpenCHAMI Phase 1 result — + OpenCHAMI Phase 2 result — rollback_needed={{ rollback_needed | default(true) }}, failed={{ openchami_rollback_failed | default(false) }} diff --git a/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml b/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml index 937601d340..bcf1b42f02 100644 --- a/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml +++ b/rollback/roles/rollback_openchami/tasks/normalize_permissions.yml @@ -24,12 +24,21 @@ # "permission denied" when the rollback later reads the backup or restores # /etc/openchami — which makes the final OpenCHAMI rollback step fail. # -# This task normalizes permissions BEFORE the restore runs: +# Additionally, SELinux contexts may prevent container access to files. +# Files need the container_file_t context to be accessible from containers. +# +# This task normalizes permissions and SELinux contexts BEFORE the restore: # - Directories -> 0755 # - Files -> 0644 +# - SELinux -> system_u:object_r:container_file_t:s0 # for two roots: -# 1. Backup root on the core container : {{ rollback_backup_dir }}/openchami -# 2. /etc/openchami on the OIM host : {{ openchami_etc_dir }} +# 1. Backup root on OIM host: {{ rollback_oim_host_backup_dir }}/openchami +# 2. /etc/openchami on OIM host: {{ openchami_etc_dir }} +# +# NOTE: All operations run on the OIM host (delegated) because: +# - The backup is on shared NFS storage mounted on OIM +# - NFS root_squash causes permission issues when accessed from core container +# - SELinux contexts must be set on the OIM host filesystem # # Behaviour: # - If permissions are simply wrong, they are corrected automatically and @@ -42,19 +51,24 @@ - name: Normalize backup and /etc/openchami permissions block: - # ── Resolve the two roots to normalize ────────────────────────────── - - name: Set permission normalization targets + # ── Resolve the two roots to normalize (OIM host paths) ─────────────── + - name: Set permission normalization targets (OIM host paths) ansible.builtin.set_fact: - normalize_backup_root: "{{ rollback_backup_dir }}/openchami" + normalize_backup_root: "{{ rollback_oim_host_backup_dir }}/openchami" normalize_etc_root: "{{ openchami_etc_dir }}" - # ── 1. Backup root (core container — local, on shared NFS path) ────── - - name: Check backup root exists (core container) + # ── 1. Backup root (OIM host — on shared NFS path) ──────────────────── + # NOTE: Must run on OIM host because NFS root_squash prevents core + # container from accessing/modifying files with restrictive permissions. + - name: Check backup root exists (OIM host) ansible.builtin.stat: path: "{{ normalize_backup_root }}" register: normalize_backup_root_stat + delegate_to: oim + delegate_facts: true + connection: ssh - - name: Normalize permissions on backup root (core container) + - name: Normalize permissions on backup root (OIM host) ansible.builtin.shell: | set -o pipefail find "{{ normalize_backup_root }}" -type d -exec chmod {{ dir_permissions_755 }} {} + 2>/dev/null || true @@ -63,8 +77,22 @@ changed_when: true failed_when: false when: normalize_backup_root_stat.stat.exists | default(false) + delegate_to: oim + delegate_facts: true + connection: ssh - - name: Detect unreadable items under backup root (core container) + - name: Fix SELinux context on backup root (OIM host) + ansible.builtin.command: > + chcon -R system_u:object_r:container_file_t:s0 "{{ normalize_backup_root }}" + register: normalize_backup_selinux + changed_when: true + failed_when: false + when: normalize_backup_root_stat.stat.exists | default(false) + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Detect unreadable items under backup root (OIM host) ansible.builtin.shell: | set -o pipefail { @@ -75,6 +103,9 @@ changed_when: false failed_when: false when: normalize_backup_root_stat.stat.exists | default(false) + delegate_to: oim + delegate_facts: true + connection: ssh # ── 2. /etc/openchami (OIM host — delegated) ──────────────────────── - name: Check /etc/openchami exists (OIM host) @@ -98,6 +129,17 @@ delegate_facts: true connection: ssh + - name: Fix SELinux context on /etc/openchami (OIM host) + ansible.builtin.command: > + chcon -R system_u:object_r:container_file_t:s0 "{{ normalize_etc_root }}" + register: normalize_etc_selinux + changed_when: true + failed_when: false + when: normalize_etc_root_stat.stat.exists | default(false) + delegate_to: oim + delegate_facts: true + connection: ssh + - name: Detect unreadable items under /etc/openchami (OIM host) ansible.builtin.shell: | set -o pipefail diff --git a/rollback/roles/rollback_openchami/vars/main.yml b/rollback/roles/rollback_openchami/vars/main.yml index 73cf84e88e..b46541de7f 100644 --- a/rollback/roles/rollback_openchami/vars/main.yml +++ b/rollback/roles/rollback_openchami/vars/main.yml @@ -133,7 +133,7 @@ rollback_messages: Rollback is not needed. Skipping. backup_found: "Backup directory found. Proceeding with rollback." permissions: - normalized: "Backup and /etc/openchami permissions verified (directories 0755, files 0644)." + normalized: "Backup and /etc/openchami permissions and SELinux contexts verified (directories 0755, files 0644, context container_file_t)." unfixable: | ════════════════════════════════════════════ OPENCHAMI ROLLBACK BLOCKED — PERMISSION ISSUE @@ -141,26 +141,31 @@ rollback_messages: One or more files/directories in the backup or /etc/openchami could not be made readable. The rollback cannot read these to restore them. - This usually happens on the shared NFS backup path (root_squash) when - container-created files carry restrictive ownership/permissions. + This usually happens due to: + 1. SELinux context issues (files need container_file_t context) + 2. NFS root_squash with restrictive ownership/permissions - Fix the permissions MANUALLY, then re-run ONLY the OpenCHAMI/OIM rollback: + Fix the permissions and SELinux context MANUALLY on the OIM host, + then re-run ONLY the OpenCHAMI/OIM rollback: - On the OIM host (as a user that owns the files, e.g. the NFS owner): + On the OIM host (SSH to the OIM, NOT the core container): + # Fix permissions chmod -R u+rwX,go+rX /etc/openchami - chmod -R u+rwX,go+rX {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami + chmod -R u+rwX,go+rX {{ rollback_oim_host_backup_dir }}/openchami - Ensure all directories are 0755 and all files are 0644, and that none - are in a permission-denied / immutable state. Then retry: + # Fix SELinux context + chcon -R system_u:object_r:container_file_t:s0 /etc/openchami + chcon -R system_u:object_r:container_file_t:s0 {{ rollback_oim_host_backup_dir }}/openchami + Then retry from the core container: cd /opt/omnia/... && ansible-playbook rollback/rollback.yml --tags oim - Relevant directories that must be readable: + Relevant directories on OIM host that must be readable: /etc/openchami, /etc/openchami/configs, /etc/openchami/pg-init - {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami - {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami/configs - {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/etc_openchami/pg-init - {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/openchami/postgresql_backup + {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami + {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami/configs + {{ rollback_oim_host_backup_dir }}/openchami/etc_openchami/pg-init + {{ rollback_oim_host_backup_dir }}/openchami/postgresql_backup restore: quadlets_success: "Restored v2.1 quadlet files from backup." quadlets_failure: "Failed to restore v2.1 quadlet files." diff --git a/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml b/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml new file mode 100644 index 0000000000..bc7d1dec53 --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml @@ -0,0 +1,99 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Cleanup Pulp Data After Rollback +# +# - Triggers Pulp DB orphan cleanup + removes filesystem orphans +# via pulp_fs_orphan_cleanup module (reads creds from cli.toml) +# - Cleans up redundant PG 16 data from backup +# - Removes temporary files from pgsql directory +# =========================================================================== + +# ── Orphan cleanup (DB-level + filesystem-level) ──────────────────────────── +# Runs inside omnia_core where cli.toml and media directory are accessible. +# No password arguments — credentials read from /root/.config/pulp/cli.toml. +- name: Run Pulp orphan cleanup (DB + filesystem) + pulp_fs_orphan_cleanup: + media_dir: "{{ pulp_data_base_path }}/pulp_storage/media" + trigger_db_orphan_cleanup: true + register: fs_orphan_result + failed_when: false + +- name: Display orphan cleanup result + ansible.builtin.debug: + msg: + - "DB orphan cleanup: {{ fs_orphan_result.db_orphan_cleanup_status | default('N/A') }}" + - "Disk artifacts: {{ fs_orphan_result.disk_artifact_count | default('N/A') }}" + - "DB artifacts: {{ fs_orphan_result.db_artifact_count | default('N/A') }}" + - "Orphans removed: {{ fs_orphan_result.removed_count | default(0) }}" + - "Space freed: {{ fs_orphan_result.freed_mb | default(0) }} MB" + +# ── Clean up backup and temp data ─────────────────────────────────────────── +- name: Clean up backup and temporary PostgreSQL data + ansible.builtin.shell: | + set -o pipefail + + BACKUP="{{ oim_host_pgsql_backup }}" + LIVE="{{ oim_host_pgsql_path }}" + + # Clean backup: remove PG 16 data, rename data_old to data + if [ -d "$BACKUP/data" ]; then + BK_VER=$(cat "$BACKUP/data/PG_VERSION" 2>/dev/null || echo "unknown") + if [ "$BK_VER" != "12" ] && [ "$BK_VER" != "13" ]; then + rm -rf "$BACKUP/data" + echo "Removed PG $BK_VER data from backup" + DATA_OLD=$(find "$BACKUP" -maxdepth 1 -type d -name "data_old.*" 2>/dev/null | head -1) + if [ -n "$DATA_OLD" ] && [ -d "$DATA_OLD" ]; then + mv "$DATA_OLD" "$BACKUP/data" + echo "Renamed $(basename "$DATA_OLD") to data in backup" + fi + fi + fi + + # Clean live pgsql: remove data_old.* and data_pg16* directories + find "$LIVE" -maxdepth 1 -type d \( -name "data_old.*" -o -name "data_pg16*" \) -exec rm -rf {} \; 2>/dev/null + echo "Cleaned up temporary directories" + args: + executable: /bin/bash + delegate_to: oim + delegate_facts: true + connection: ssh + register: cleanup_result + changed_when: true + failed_when: false + +- name: Display cleanup result + ansible.builtin.debug: + msg: "{{ cleanup_result.stdout_lines | default(['Cleanup completed']) }}" + +# ── Report disk usage ────────────────────────────────────────────────────── +- name: Report disk usage after rollback + ansible.builtin.shell: | + echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)" + echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)" + echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)" + args: + executable: /bin/bash + delegate_to: oim + delegate_facts: true + connection: ssh + register: disk_usage_result + changed_when: false + failed_when: false + +- name: Display disk usage + ansible.builtin.debug: + msg: "{{ disk_usage_result.stdout_lines | default([]) }}" diff --git a/rollback/roles/rollback_pulp/tasks/main.yml b/rollback/roles/rollback_pulp/tasks/main.yml new file mode 100644 index 0000000000..ba34c235d2 --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/main.yml @@ -0,0 +1,88 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# rollback_pulp — Main Orchestration +# ============================================================================ +# Reverse the Pulp upgrade (3.113 → 3.80) by: +# 1. Resolving admin NIC IP for health checks +# 2. Pre-rollback checks (verify Pulp is deployed, backup exists) +# 3. Stopping the current Pulp container +# 4. Restoring PostgreSQL data from backup (PG 12/13 compatible) +# 5. Restoring the v2.1 quadlet file from backup +# 6. Pulling the v2.1 Pulp image (3.80) +# 7. Starting the rolled-back container +# 8. Post-rollback health check +# 9. Cleanup (orphan artifacts, redundant backup data) +# +# IMPORTANT: PostgreSQL data must be restored because: +# - Pulp 3.80 uses PostgreSQL 12/13 +# - Pulp 3.113 uses PostgreSQL 16 +# - PostgreSQL cannot downgrade data files between major versions +# +# No idempotency check — rollback always executes regardless of current +# container state. This ensures rollback works even in partial or failed +# upgrade states. +# ============================================================================ + +- name: Read oim_metadata.yml for shared storage path + ansible.builtin.slurp: + src: "{{ oim_metadata_path }}" + register: _pulp_oim_metadata_raw + +- name: Set OIM host-side paths from metadata + ansible.builtin.set_fact: + oim_shared_path: "{{ (_pulp_oim_metadata_raw.content | b64decode | from_yaml).oim_shared_path | regex_replace('/$', '') }}" + +- name: Derive OIM host-side Pulp paths + ansible.builtin.set_fact: + oim_host_pulp_data_base: "{{ oim_shared_path }}/omnia/pulp/settings" + oim_host_pgsql_path: "{{ oim_shared_path }}/omnia/pulp/settings/pgsql" + oim_host_backup_dir: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp" + oim_host_pgsql_backup: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp/pgsql" + oim_host_rollback_backup_dir: >- + {{ (rollback_backup_dir | default(rollback_backup_dir_default)) + | regex_replace('^/opt/omnia', oim_shared_path ~ '/omnia') }} + +- name: Resolve admin NIC IP for Pulp API endpoints + ansible.builtin.include_tasks: resolve_admin_ip.yml + +- name: Pulp rollback workflow + block: + - name: Pre-rollback validation checks + ansible.builtin.include_tasks: pre_rollback_checks.yml + + - name: Execute Pulp rollback + ansible.builtin.include_tasks: rollback_pulp_container.yml + when: pulp_deployed | default(false) | bool + + - name: Post-rollback health check + ansible.builtin.include_tasks: post_rollback_health_check.yml + when: pulp_deployed | default(false) | bool + + - name: Post-rollback cleanup (orphans and temporary data) + ansible.builtin.include_tasks: cleanup_pulp_data.yml + when: + - pulp_deployed | default(false) | bool + - pulp_rollback_success | default(false) | bool + + rescue: + - name: Set rollback failure flag + ansible.builtin.set_fact: + pulp_rollback_failed: true + + always: + - name: Rollback status and cleanup + ansible.builtin.include_tasks: rollback_status.yml diff --git a/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml b/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml new file mode 100644 index 0000000000..dec934a8db --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/post_rollback_health_check.yml @@ -0,0 +1,91 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# post_rollback_health_check.yml — Verify Pulp Rollback Success +# +# Validates: +# 1. Pulp container is running with the correct (rolled-back) image +# 2. Pulp API is accessible and responding +# Sets pulp_rollback_success flag for cleanup tasks. +# ============================================================================ + +- name: Post-rollback health check + block: + # ── Verify container is running with correct image ──────────────────── + - name: Get rolled-back Pulp container info + containers.podman.podman_container_info: + name: "{{ pulp_container_name }}" + register: pulp_post_info + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Verify Pulp is running with rolled-back image + ansible.builtin.assert: + that: + - pulp_post_info.containers | length > 0 + - pulp_post_info.containers[0].State.Status == 'running' + - pulp_rollback_tag in (pulp_post_info.containers[0].ImageName | default('')) + fail_msg: | + Pulp rollback verification failed. + Expected image tag: {{ pulp_rollback_tag }} + Actual image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }} + Container status: {{ pulp_post_info.containers[0].State.Status | default('unknown') }} + success_msg: "Pulp container is running with rolled-back image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }}" + + # ── Verify Pulp API is accessible ───────────────────────────────────── + - name: Check Pulp API status after rollback + ansible.builtin.uri: + url: "{{ pulp_status_url }}" + method: GET + validate_certs: false + status_code: [200] + return_content: true + register: pulp_post_health + retries: "{{ pulp_health_retries }}" + delay: "{{ pulp_health_delay }}" + until: pulp_post_health.status == 200 + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Display Pulp workers status + ansible.builtin.debug: + msg: >- + Pulp API: healthy (HTTP {{ pulp_post_health.status }}) | + Workers: {{ (pulp_post_health.json | default({})).online_workers | default([]) | length }} | + Content apps: {{ (pulp_post_health.json | default({})).online_content_apps | default([]) | length }} + + - name: Set rollback success flag + ansible.builtin.set_fact: + pulp_rollback_success: true + + - name: Display rollback success message + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.rollback_success }}" + + rescue: + - name: Display rollback health check failure + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.post_check_failure }}" + + - name: Set rollback failure flag + ansible.builtin.set_fact: + pulp_rollback_failed: true + + - name: Fail with health check error + ansible.builtin.fail: + msg: "Pulp post-rollback health check failed. Check container logs: podman logs {{ pulp_container_name }}" diff --git a/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml new file mode 100644 index 0000000000..59df22ca94 --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml @@ -0,0 +1,134 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# pre_rollback_checks.yml — Validate Pulp Rollback Preconditions +# ============================================================================ +# Checks: +# 1. Pulp container exists and is deployed +# 2. Backup directory exists with Pulp backup +# 3. Backed-up quadlet file or image tag exists +# +# No idempotency check — rollback always executes regardless of current +# container state. This ensures rollback works even in partial or failed +# upgrade states. +# ============================================================================ + +- name: Pre-rollback validation + block: + # ── Check if Pulp container exists ──────────────────────────────────── + - name: Check if Pulp container exists + ansible.builtin.command: podman container exists {{ pulp_container_name }} + register: pulp_exists_check + changed_when: false + failed_when: false + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Set Pulp deployment status + ansible.builtin.set_fact: + pulp_deployed: "{{ pulp_exists_check.rc == 0 }}" + + - name: Display Pulp not deployed message + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.not_deployed }}" + when: not pulp_deployed + + # ── Get current Pulp container info ─────────────────────────────────── + - name: Get current Pulp container info + containers.podman.podman_container_info: + name: "{{ pulp_container_name }}" + register: pulp_pre_info + delegate_to: oim + delegate_facts: true + connection: ssh + when: pulp_deployed + + - name: Extract current Pulp image version + ansible.builtin.set_fact: + pulp_current_image: "{{ pulp_pre_info.containers[0].ImageName | default('unknown') }}" + pulp_current_status: "{{ pulp_pre_info.containers[0].State.Status | default('unknown') }}" + when: + - pulp_deployed + - pulp_pre_info.containers | length > 0 + + - name: Display current Pulp version + ansible.builtin.debug: + msg: "Current Pulp image: {{ pulp_current_image | default('not found') }}, Status: {{ pulp_current_status | default('unknown') }}" + when: pulp_deployed + + # ── Check if already at rollback target version ─────────────────────── + - name: Check if Pulp is already at rollback target version + ansible.builtin.set_fact: + pulp_already_rolled_back: "{{ pulp_rollback_image in (pulp_current_image | default('')) }}" + when: pulp_deployed + + - name: Display skip message if already at target version + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.already_v21 }}" + when: + - pulp_deployed + - pulp_already_rolled_back | default(false) + + # ── Verify backup directory exists ──────────────────────────────────── + - name: Check backup directory exists + ansible.builtin.stat: + path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}" + register: rollback_pulp_backup_stat + when: pulp_deployed + + - name: Check backed-up quadlet file exists + ansible.builtin.stat: + path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_quadlet_subpath }}/{{ pulp_container_name }}.container" + register: rollback_pulp_quadlet_stat + when: pulp_deployed + + - name: Check backed-up image tag file exists + ansible.builtin.stat: + path: "{{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_image_tag_subpath }}" + register: rollback_pulp_image_tag_stat + when: pulp_deployed + + - name: Display backup inventory + ansible.builtin.debug: + verbosity: 1 + msg: + - "Backup directory: {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}" + - "Pulp backup: {{ 'found' if rollback_pulp_backup_stat.stat.exists | default(false) else 'MISSING' }}" + - "Quadlet backup: {{ 'found' if rollback_pulp_quadlet_stat.stat.exists | default(false) else 'MISSING' }}" + - "Image tag backup: {{ 'found' if rollback_pulp_image_tag_stat.stat.exists | default(false) else 'MISSING' }}" + when: pulp_deployed + + # ── Determine rollback strategy ─────────────────────────────────────── + # If quadlet backup exists, restore it. Otherwise, update image tag in current quadlet. + - name: Determine rollback strategy + ansible.builtin.set_fact: + pulp_rollback_strategy: >- + {{ 'restore_quadlet' if (rollback_pulp_quadlet_stat.stat.exists | default(false)) + else 'update_image_tag' }} + when: pulp_deployed + + - name: Display rollback strategy + ansible.builtin.debug: + msg: "Pulp rollback strategy: {{ pulp_rollback_strategy | default('N/A') }}" + when: pulp_deployed + + - name: Display rollback proceeding message + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.backup_found }}" + when: + - pulp_deployed + - rollback_pulp_backup_stat.stat.exists | default(false) or not (pulp_already_rolled_back | default(false)) diff --git a/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml b/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml new file mode 100644 index 0000000000..339b864d0d --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/resolve_admin_ip.yml @@ -0,0 +1,95 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# resolve_admin_ip.yml — Resolve Admin NIC IP for Pulp API Health Checks +# ============================================================================ +# Resolves the admin_nic_ip needed for Pulp API health check endpoints. +# +# Resolution order: +# 1. Primary: Extract from local_repo_access.yml (same as upgrade_k8s) +# 2. Fallback: Extract from network_spec.yml (same as prepare_oim) +# 3. Last resort: Use localhost +# +# Sets fact: +# admin_nic_ip — IP address for Pulp API health checks +# ============================================================================ + +- name: Resolve admin NIC IP for Pulp health checks + block: + # Primary method: Extract from local_repo_access.yml (same as upgrade_k8s) + - name: Check if local_repo_access.yml exists + ansible.builtin.stat: + path: "{{ local_repo_access_path }}" + register: local_repo_access_stat + + - name: Load local_repo_access.yml + ansible.builtin.slurp: + src: "{{ local_repo_access_path }}" + register: local_repo_access_raw + when: local_repo_access_stat.stat.exists + + - name: Parse local_repo_access.yml + ansible.builtin.set_fact: + _local_repo_access: "{{ local_repo_access_raw.content | b64decode | from_yaml }}" + when: local_repo_access_stat.stat.exists + + - name: Set admin_nic_ip from local_repo_access + ansible.builtin.set_fact: + admin_nic_ip: "{{ _local_repo_access.offline_tarball_path | regex_replace('^(https?)://([^:]+):.*', '\\2') }}" + when: + - local_repo_access_stat.stat.exists + - _local_repo_access.offline_tarball_path is defined + + # Fallback method: Extract from network_spec.yml + - name: Check if network_spec.yml exists + ansible.builtin.stat: + path: "{{ network_spec_path }}" + register: network_spec_stat + when: admin_nic_ip is not defined + + - name: Load network_spec.yml as fallback + ansible.builtin.include_vars: + file: "{{ network_spec_path }}" + when: + - admin_nic_ip is not defined + - network_spec_stat.stat.exists | default(false) + + # Networks is a list in network_spec.yml, parse it to extract admin_network + - name: Parse network_spec data into network_data + ansible.builtin.set_fact: + network_data: "{{ network_data | default({}) | combine({item.keys() | first: item.values() | first}) }}" + loop: "{{ Networks }}" + when: + - admin_nic_ip is not defined + - Networks is defined + + - name: Set admin_nic_ip from network_spec + ansible.builtin.set_fact: + admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}" + when: + - admin_nic_ip is not defined + - network_data is defined + - network_data.admin_network is defined + - network_data.admin_network.primary_oim_admin_ip is defined + + rescue: + - name: Fallback to localhost for admin_nic_ip + ansible.builtin.set_fact: + admin_nic_ip: "localhost" + +- name: Display resolved admin_nic_ip + ansible.builtin.debug: + msg: "Pulp rollback health checks will use admin_nic_ip: {{ admin_nic_ip }}" diff --git a/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml b/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml new file mode 100644 index 0000000000..c075481ce4 --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/restore_pulp_data.yml @@ -0,0 +1,41 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Restore Pulp PostgreSQL Data During Rollback +# +# Uses the pulp_pgsql_backup_restore module to intelligently restore PG 12/13 +# data from the backup, handling the case where the backup may contain +# PG 16 data (from the upgrade) with the original PG 12/13 data in +# a data_old.* directory. +# +# Runs inside omnia_core where backup and pgsql paths are accessible +# via shared NFS storage at /opt/omnia/. +# =========================================================================== + +- name: Restore Pulp PostgreSQL data for rollback + pulp_pgsql_backup_restore: + action: restore + backup_path: "{{ pulp_pgsql_backup_path }}" + dest_path: "{{ pulp_pgsql_path }}" + register: pgsql_restore_result + +- name: Display restore result + ansible.builtin.debug: + msg: + - "Restore mode: {{ pgsql_restore_result.restore_mode }}" + - "Backup PG version: {{ pgsql_restore_result.backup_pg_version }}" + - "Restored PG version: {{ pgsql_restore_result.restored_pg_version }}" + - "{{ pgsql_restore_result.messages | join(', ') }}" diff --git a/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml new file mode 100644 index 0000000000..cb74c969aa --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/rollback_pulp_container.yml @@ -0,0 +1,131 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# Pulp Container Rollback (3.113 → 3.80) +# +# Steps: +# 1. Pull the v2.1 Pulp image (3.80) +# 2. Stop and remove current container +# 3. Restore PostgreSQL data (PG 16 → PG 12/13) +# 4. Update quadlet file with rollback image tag +# 5. Reload systemd, start container, wait for init +# ============================================================================ + +- name: Skip rollback if already at target version + ansible.builtin.debug: + msg: "Pulp is already at rollback target version. Skipping." + when: pulp_already_rolled_back | default(false) + +- name: Execute Pulp container rollback + when: not (pulp_already_rolled_back | default(false)) + block: + # --- 1. Pull v2.1 Pulp image --- + - name: Pull v2.1 Pulp image ({{ pulp_rollback_image }}) + containers.podman.podman_image: + name: "{{ pulp_rollback_image }}" + state: present + force: true + register: pulp_image_pull + retries: "{{ pull_image_retries }}" + delay: "{{ pull_image_delay }}" + until: pulp_image_pull is succeeded + delegate_to: oim + delegate_facts: true + connection: ssh + + # --- 2. Stop and remove current container --- + - name: Stop Pulp systemd service + ansible.builtin.systemd: + name: "{{ pulp_container_name }}" + state: stopped + delegate_to: oim + delegate_facts: true + connection: ssh + failed_when: false + + - name: Force remove Pulp container + ansible.builtin.command: podman rm -f {{ pulp_container_name }} + delegate_to: oim + delegate_facts: true + connection: ssh + changed_when: true + failed_when: false + + # --- 3. Restore PostgreSQL data from backup --- + - name: Restore Pulp PostgreSQL data from backup + ansible.builtin.include_tasks: restore_pulp_data.yml + + # --- 4. Update quadlet file with rollback image --- + - name: Restore quadlet file from backup + ansible.builtin.copy: + src: "{{ oim_host_rollback_backup_dir }}/{{ backup_pulp_quadlet_subpath }}/{{ pulp_container_name }}.container" + dest: "{{ pulp_quadlet_path }}" + mode: "{{ file_permissions_644 }}" + remote_src: true + delegate_to: oim + delegate_facts: true + connection: ssh + when: pulp_rollback_strategy | default('update_image_tag') == 'restore_quadlet' + + - name: Update Image line in quadlet file + ansible.builtin.replace: + path: "{{ pulp_quadlet_path }}" + regexp: '^Image=.*pulp.*$' + replace: "Image={{ pulp_rollback_image }}" + delegate_to: oim + delegate_facts: true + connection: ssh + when: pulp_rollback_strategy | default('update_image_tag') == 'update_image_tag' + + # --- 5. Reload systemd, start container, wait for init --- + - name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Start Pulp systemd service + ansible.builtin.systemd: + name: "{{ pulp_container_name }}" + state: started + enabled: true + register: pulp_start_result + retries: "{{ pulp_startup_retries }}" + delay: "{{ pulp_startup_delay }}" + until: pulp_start_result is succeeded + delegate_to: oim + delegate_facts: true + connection: ssh + + - name: Wait for Pulp container to initialize + ansible.builtin.pause: + seconds: "{{ pulp_init_wait }}" + + - name: Verify Pulp container is running + ansible.builtin.command: podman ps --filter name={{ pulp_container_name }} --format "{{ '{{' }}.Status{{ '}}' }}" + register: pulp_status_check + delegate_to: oim + delegate_facts: true + connection: ssh + changed_when: false + retries: "{{ pulp_startup_retries }}" + delay: "{{ pulp_startup_delay }}" + until: "'Up' in pulp_status_check.stdout" + + - name: Display Pulp container status + ansible.builtin.debug: + msg: "Pulp container status: {{ pulp_status_check.stdout }}" diff --git a/rollback/roles/rollback_pulp/tasks/rollback_status.yml b/rollback/roles/rollback_pulp/tasks/rollback_status.yml new file mode 100644 index 0000000000..35fb823f9c --- /dev/null +++ b/rollback/roles/rollback_pulp/tasks/rollback_status.yml @@ -0,0 +1,52 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# rollback_status.yml — Report Pulp Rollback Status +# ============================================================================ +# Reports the final status of the Pulp rollback operation. +# This task runs in the 'always' block to ensure status is reported +# regardless of success or failure. +# ============================================================================ + +- name: Determine final rollback status + ansible.builtin.set_fact: + pulp_rollback_final_status: >- + {{ 'skipped' if not (pulp_deployed | default(false)) + else ('skipped' if (pulp_already_rolled_back | default(false)) + else ('failed' if (pulp_rollback_failed | default(false)) + else 'completed')) }} + +- name: Display Pulp rollback summary + ansible.builtin.debug: + msg: + - "════════════════════════════════════════════════════════════" + - " PULP ROLLBACK STATUS: {{ pulp_rollback_final_status | upper }}" + - "════════════════════════════════════════════════════════════" + - "Deployed: {{ pulp_deployed | default(false) }}" + - "Already at target: {{ pulp_already_rolled_back | default(false) }}" + - "Rollback failed: {{ pulp_rollback_failed | default(false) }}" + - "Target image: {{ pulp_rollback_image }}" + - "════════════════════════════════════════════════════════════" + +- name: Display failure message if rollback failed + ansible.builtin.debug: + msg: "{{ rollback_messages.pulp.rollback_failure }}" + when: pulp_rollback_failed | default(false) + +- name: Re-raise failure if rollback failed + ansible.builtin.fail: + msg: "Pulp rollback failed. See above for details." + when: pulp_rollback_failed | default(false) diff --git a/rollback/roles/rollback_pulp/vars/main.yml b/rollback/roles/rollback_pulp/vars/main.yml new file mode 100644 index 0000000000..21ff282c46 --- /dev/null +++ b/rollback/roles/rollback_pulp/vars/main.yml @@ -0,0 +1,120 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# rollback_pulp — Variables +# ============================================================================ +# Rollback target: Pulp 3.80 (from Pulp 3.113) +# Reverses the upgrade performed by upgrade_pulp role. +# ============================================================================ + +# File permissions +dir_permissions_755: "0755" +file_permissions_644: "0644" + +# Manifest path for locating backup directory +manifest_path: "/opt/omnia/.data/upgrade_manifest.yml" + +# Default backup directory (must match upgrade_pulp) +rollback_backup_dir_default: "/opt/omnia/backups/upgrade/version_2.1.0.0" + +# Pulp container settings +pulp_container_name: "pulp" +pulp_rollback_tag: "3.80" +pulp_rollback_image: "docker.io/pulp/pulp:{{ pulp_rollback_tag }}" + +# Pulp quadlet file path +pulp_quadlet_path: "/etc/containers/systemd/{{ pulp_container_name }}.container" + +# OIM metadata (for resolving host-side shared storage paths) +oim_metadata_path: "/opt/omnia/.data/oim_metadata.yml" + +# Paths for resolving admin_nic_ip +local_repo_access_path: "/opt/omnia/provision/local_repo_access.yml" +network_spec_path: "{{ input_project_dir | default('/opt/omnia/input') }}/network_spec.yml" + +# Pulp API endpoint for health checks +pulp_protocol: "https" +pulp_port: "2225" +pulp_status_url: "{{ pulp_protocol }}://{{ admin_nic_ip | default('localhost') }}:{{ pulp_port }}/pulp/api/v3/status/" + +# Image pull settings +pull_image_retries: 5 +pull_image_delay: 10 + +# Pulp startup and health check settings +pulp_startup_retries: 10 +pulp_startup_delay: 10 +pulp_init_wait: 30 +pulp_health_retries: 12 +pulp_health_delay: 10 + +# Backup sub-paths (relative to rollback_backup_dir) +backup_pulp_subpath: "pulp" +backup_pulp_quadlet_subpath: "pulp/quadlet" +backup_pulp_image_tag_subpath: "pulp/image_tag.txt" + +# Pulp shared storage paths (OIM container paths) +# These paths are inside the OIM container and mounted as volumes in the Pulp container +pulp_data_base_path: "/opt/omnia/pulp/settings" +pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" + +# PostgreSQL backup paths +# PostgreSQL data must be restored during rollback because: +# - Pulp 3.80 uses PostgreSQL 12/13 +# - Pulp 3.113 uses PostgreSQL 16 +# - PostgreSQL cannot downgrade data files between major versions +pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" +pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" + +# Rollback messages +rollback_messages: + pulp: + not_deployed: | + Pulp container is not deployed on the OIM. Skipping Pulp rollback. + no_backup: | + FATAL: Pulp backup not found at {{ rollback_backup_dir | default(rollback_backup_dir_default) }}/{{ backup_pulp_subpath }}. + Cannot rollback without a valid backup. + already_v21: | + Pulp is already at v2.1 version ({{ pulp_rollback_image }}). Skipping rollback. + backup_found: "Pulp backup found. Proceeding with rollback." + rollback_success: | + ════════════════════════════════════════════════════════════ + PULP ROLLBACK COMPLETED SUCCESSFULLY + ════════════════════════════════════════════════════════════ + Previous image: {{ pulp_current_image | default('unknown') }} + Rolled back to: {{ pulp_rollback_image }} + Container: {{ pulp_container_name }} + Status: running + API endpoint: {{ pulp_status_url }} + ════════════════════════════════════════════════════════════ + rollback_failure: | + ════════════════════════════════════════════════════════════ + PULP ROLLBACK FAILED + ════════════════════════════════════════════════════════════ + Target image: {{ pulp_rollback_image }} + + Please check: + - Pulp container logs: podman logs {{ pulp_container_name }} + - Pulp service status: systemctl status {{ pulp_container_name }} + - Shared storage accessibility + ════════════════════════════════════════════════════════════ + pre_check_failure: | + Pulp pre-rollback check failed. + Please verify Pulp container state before attempting rollback. + post_check_failure: | + Pulp post-rollback health check failed. + The rolled-back Pulp container may not be functioning correctly. + Check container logs: podman logs {{ pulp_container_name }} diff --git a/upgrade/playbooks/upgrade_oim.yml b/upgrade/playbooks/upgrade_oim.yml index 5342650777..65ff749392 100644 --- a/upgrade/playbooks/upgrade_oim.yml +++ b/upgrade/playbooks/upgrade_oim.yml @@ -15,18 +15,19 @@ # ============================================================================ # upgrade_oim.yml — Internal playbook (imported by upgrade.yml --tags oim) # ============================================================================ -# Upgrades OIM components: OpenCHAMI containers. +# Upgrades OIM components: Pulp container and OpenCHAMI containers. # Prerequisites: prepare_upgrade.yml must have been run first. # Reads upgrade_manifest.yml and skips if oim already completed. # # Flow: # 1. Pre-flight: read manifest, check idempotency -# 2. OpenCHAMI container upgrade (pg_dump, deployment-recipes, image pull, -# ordered restart, DB migration, validation) -# 3. Mark OIM as completed in manifest +# 2. Phase 1: Pulp container upgrade (3.80 → 3.113) +# 3. Phase 2: OpenCHAMI container upgrade (pg_dump, deployment-recipes, +# image pull, ordered restart, DB migration, validation) +# 4. Mark OIM as completed in manifest # ============================================================================ -- name: Upgrade OIM (OpenCHAMI) +- name: Upgrade OIM (Pulp and OpenCHAMI) hosts: localhost connection: local gather_facts: true @@ -64,7 +65,55 @@ ansible.builtin.debug: msg: "[UPGRADE] Component '{{ component_name }}' — status changed to: in-progress" - # ── Phase 1: OpenCHAMI Container Upgrade (ESpec §4.4) ────────────── + # ── Phase 1: Pulp Container Upgrade ───────────────────────────────── + # The upgrade_pulp role handles the full lifecycle: + # resolve_admin_ip → pre_upgrade_health_check → upgrade_pulp_container → post_upgrade_health_check + # It delegates all container operations to the 'oim' host via SSH. + # Pulp data is preserved on shared storage during the upgrade. + - name: "Phase 1 — Pulp Container Upgrade" + block: + - name: Upgrade Pulp container + ansible.builtin.include_role: + name: "{{ playbook_dir }}/../roles/upgrade_pulp" + + - name: Display Pulp upgrade outcome + ansible.builtin.debug: + msg: >- + Pulp Phase 1 result — + deployed={{ pulp_deployed | default(false) }}, + upgrade_needed={{ pulp_upgrade_needed | default(true) }} + + rescue: + - name: Re-read upgrade_manifest.yml after Pulp failure + ansible.builtin.slurp: + src: "{{ manifest_path }}" + register: pulp_rescue_raw_manifest + + - name: Parse current manifest state + ansible.builtin.set_fact: + pulp_rescue_manifest: "{{ pulp_rescue_raw_manifest.content | b64decode | from_yaml }}" + + - name: Pulp upgrade failed — mark component as failed + ansible.builtin.copy: + content: >- + {{ pulp_rescue_manifest | combine({ + 'component_status': pulp_rescue_manifest.component_status | combine({ + component_name: 'failed' + }) + }) | to_nice_yaml }} + dest: "{{ manifest_path }}" + mode: '0644' + + - name: Fail with Pulp upgrade error + ansible.builtin.fail: + msg: >- + Pulp upgrade failed + ({{ pulp_rescue_manifest.source_version | default('N/A') }} → + {{ pulp_rescue_manifest.target_version | default('N/A') }}). + Check logs: podman logs pulp + Consider running rollback. + + # ── Phase 2: OpenCHAMI Container Upgrade (ESpec §4.4) ────────────── # The upgrade_openchami role handles the full lifecycle: # pre_upgrade_health_check → upgrade_openchami_containers → post_upgrade_health_check # It delegates all container operations to the 'oim' host via SSH. @@ -72,7 +121,7 @@ # which lands in the rescue block below. # On skip (not deployed / already at target), the role completes normally # and sets openchami_deployed / upgrade_needed facts accordingly. - - name: "Phase 1 — OpenCHAMI Container Upgrade" + - name: "Phase 2 — OpenCHAMI Container Upgrade" block: - name: Upgrade OpenCHAMI containers and services ansible.builtin.include_role: @@ -81,7 +130,7 @@ - name: Display OpenCHAMI upgrade outcome ansible.builtin.debug: msg: >- - OpenCHAMI Phase 1 result — + OpenCHAMI Phase 2 result — deployed={{ openchami_deployed | default(false) }}, upgrade_needed={{ upgrade_needed | default(true) }}, failed={{ openchami_upgrade_failed | default(false) }} diff --git a/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml new file mode 100644 index 0000000000..2b19607817 --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/backup_pulp_data.yml @@ -0,0 +1,37 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Backup Pulp PostgreSQL Data Before Upgrade +# +# PostgreSQL data must be backed up before upgrade because: +# - Pulp 3.80 uses PostgreSQL 12/13 +# - Pulp 3.113 uses PostgreSQL 16 +# - PostgreSQL cannot downgrade data files between major versions +# +# Without this backup, rollback is not possible. +# =========================================================================== + +- name: Backup Pulp PostgreSQL data + pulp_pgsql_backup_restore: + action: backup + src_path: "{{ pulp_pgsql_path }}" + dest_path: "{{ pulp_pgsql_backup_path }}" + backup_dir: "{{ pulp_backup_dir }}" + register: pulp_backup_result + +- name: Display backup result + ansible.builtin.debug: + msg: "{{ pulp_backup_result.messages | default(['Backup task completed']) }}" diff --git a/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml b/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml new file mode 100644 index 0000000000..0cac71372d --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml @@ -0,0 +1,57 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Cleanup After Pulp Upgrade +# +# - Runs Pulp orphan cleanup to reclaim disk space +# via pulp_fs_orphan_cleanup module (reads creds from cli.toml) +# - Reports disk usage +# NOTE: Backup directory is preserved for rollback. +# =========================================================================== + +# Runs inside omnia_core where cli.toml and media directory are accessible. +# No password arguments — credentials read from /root/.config/pulp/cli.toml. +- name: Run Pulp orphan cleanup after upgrade + pulp_fs_orphan_cleanup: + media_dir: "{{ pulp_data_base_path }}/pulp_storage/media" + trigger_db_orphan_cleanup: true + register: orphan_result + failed_when: false + +- name: Display orphan cleanup result + ansible.builtin.debug: + msg: + - "DB orphan cleanup: {{ orphan_result.db_orphan_cleanup_status | default('N/A') }}" + - "Orphans removed: {{ orphan_result.removed_count | default(0) }}" + - "Space freed: {{ orphan_result.freed_mb | default(0) }} MB" + +- name: Report disk usage after upgrade + ansible.builtin.shell: | + echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)" + echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)" + echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)" + args: + executable: /bin/bash + delegate_to: oim + delegate_facts: true + connection: ssh + register: disk_usage_result + changed_when: false + failed_when: false + +- name: Display disk usage + ansible.builtin.debug: + msg: "{{ disk_usage_result.stdout_lines | default([]) }}" diff --git a/upgrade/roles/upgrade_pulp/tasks/main.yml b/upgrade/roles/upgrade_pulp/tasks/main.yml new file mode 100644 index 0000000000..4ec5803ca2 --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/main.yml @@ -0,0 +1,87 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Upgrade Pulp Role - Main Entry Point +# +# This role upgrades the Pulp container from 3.80 to 3.113. +# Pulp stores all data in persistent volumes mounted from shared storage, +# so data is preserved during the upgrade. +# +# IMPORTANT: PostgreSQL data must be backed up before upgrade because: +# - Pulp 3.80 uses PostgreSQL 12/13 +# - Pulp 3.113 uses PostgreSQL 16 +# - PostgreSQL cannot downgrade data files between major versions +# Without the backup, rollback is not possible. +# +# Flow: +# 1. Resolve admin NIC IP for health check endpoints +# 2. Pre-upgrade health check (verify Pulp is deployed and accessible) +# 3. Backup PostgreSQL data (required for rollback) +# 4. Pull the new Pulp image (3.113) +# 5. Stop the Pulp container +# 6. Update the quadlet file with the new image tag +# 7. Reload systemd daemon and restart container +# 8. Run database migrations +# 9. Post-upgrade health check +# 10. Cleanup (orphan artifacts) +# =========================================================================== + +- name: Read oim_metadata.yml for shared storage path + ansible.builtin.slurp: + src: "{{ oim_metadata_path }}" + register: _pulp_oim_metadata_raw + +- name: Set OIM host-side paths from metadata + ansible.builtin.set_fact: + oim_shared_path: "{{ (_pulp_oim_metadata_raw.content | b64decode | from_yaml).oim_shared_path | regex_replace('/$', '') }}" + +- name: Derive OIM host-side Pulp paths + ansible.builtin.set_fact: + oim_host_pulp_data_base: "{{ oim_shared_path }}/omnia/pulp/settings" + oim_host_pgsql_path: "{{ oim_shared_path }}/omnia/pulp/settings/pgsql" + oim_host_backup_dir: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp" + oim_host_pgsql_backup: "{{ oim_shared_path }}/omnia/backups/upgrade/version_2.1.0.0/pulp/pgsql" + +- name: Resolve admin NIC IP for Pulp API endpoints + ansible.builtin.include_tasks: resolve_admin_ip.yml + +- name: Pre-upgrade health check + ansible.builtin.include_tasks: pre_upgrade_health_check.yml + +- name: Backup Pulp PostgreSQL data before upgrade + ansible.builtin.include_tasks: backup_pulp_data.yml + when: + - pulp_deployed | default(false) | bool + - pulp_upgrade_needed | default(true) | bool + +- name: Execute Pulp upgrade + ansible.builtin.include_tasks: upgrade_pulp_container.yml + when: + - pulp_deployed | default(false) | bool + - pulp_upgrade_needed | default(true) | bool + +- name: Post-upgrade health check + ansible.builtin.include_tasks: post_upgrade_health_check.yml + when: + - pulp_deployed | default(false) | bool + - pulp_upgrade_needed | default(true) | bool + +- name: Post-upgrade cleanup (orphan artifacts) + ansible.builtin.include_tasks: cleanup_after_upgrade.yml + when: + - pulp_deployed | default(false) | bool + - pulp_upgrade_needed | default(true) | bool + - pulp_upgrade_success | default(false) | bool diff --git a/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml b/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml new file mode 100644 index 0000000000..83f99a47d6 --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/post_upgrade_health_check.yml @@ -0,0 +1,75 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Pulp Post-Upgrade Health Check +# +# Verifies: +# 1. Pulp API is accessible and responding +# 2. Pulp container is running with the new image +# Sets pulp_upgrade_success flag for cleanup tasks. +# =========================================================================== + +- name: Check Pulp API status after upgrade + ansible.builtin.uri: + url: "{{ pulp_status_url }}" + method: GET + validate_certs: false + status_code: [200] + register: pulp_post_health + retries: "{{ pulp_health_retries }}" + delay: "{{ pulp_health_delay }}" + until: pulp_post_health.status == 200 + delegate_to: oim + delegate_facts: true + connection: ssh + +- name: Get Pulp container info after upgrade + containers.podman.podman_container_info: + name: "{{ pulp_container_name }}" + register: pulp_post_info + delegate_to: oim + delegate_facts: true + connection: ssh + +- name: Verify Pulp is running with new image + ansible.builtin.assert: + that: + - pulp_post_info.containers | length > 0 + - pulp_post_info.containers[0].State.Status == 'running' + - pulp_target_image in pulp_post_info.containers[0].ImageName + fail_msg: | + Pulp upgrade verification failed. + Expected image: {{ pulp_target_image }} + Actual image: {{ pulp_post_info.containers[0].ImageName | default('unknown') }} + Container status: {{ pulp_post_info.containers[0].State.Status | default('unknown') }} + success_msg: "Pulp successfully upgraded to {{ pulp_target_image }}" + +- name: Set upgrade success flag + ansible.builtin.set_fact: + pulp_upgrade_success: true + +- name: Display Pulp upgrade summary + ansible.builtin.debug: + msg: + - "════════════════════════════════════════════════════════════" + - " PULP UPGRADE COMPLETED SUCCESSFULLY" + - "════════════════════════════════════════════════════════════" + - " Previous image: {{ pulp_current_image | default('unknown') }}" + - " New image: {{ pulp_target_image }}" + - " Container: {{ pulp_container_name }}" + - " Status: running" + - " API endpoint: {{ pulp_status_url }}" + - "════════════════════════════════════════════════════════════" diff --git a/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml new file mode 100644 index 0000000000..5819991864 --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml @@ -0,0 +1,100 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Pulp Pre-Upgrade Health Check +# +# Verifies: +# 1. Pulp container exists and is deployed +# 2. Current Pulp version (to determine if upgrade is needed) +# 3. Pulp API is accessible +# =========================================================================== + +- name: Check if Pulp container exists + ansible.builtin.command: podman container exists {{ pulp_container_name }} + register: pulp_exists_check + changed_when: false + failed_when: false + delegate_to: oim + delegate_facts: true + connection: ssh + +- name: Set Pulp deployment status + ansible.builtin.set_fact: + pulp_deployed: "{{ pulp_exists_check.rc == 0 }}" + +- name: Display Pulp not deployed message + ansible.builtin.debug: + msg: "{{ upgrade_messages.pulp.not_deployed }}" + when: not pulp_deployed + +- name: Get current Pulp container info + containers.podman.podman_container_info: + name: "{{ pulp_container_name }}" + register: pulp_pre_info + delegate_to: oim + delegate_facts: true + connection: ssh + when: pulp_deployed + +- name: Extract current Pulp image version + ansible.builtin.set_fact: + pulp_current_image: "{{ pulp_pre_info.containers[0].ImageName | default('unknown') }}" + pulp_current_status: "{{ pulp_pre_info.containers[0].State.Status | default('unknown') }}" + when: + - pulp_deployed + - pulp_pre_info.containers | length > 0 + +- name: Display current Pulp version + ansible.builtin.debug: + msg: "Current Pulp image: {{ pulp_current_image | default('not found') }}, Status: {{ pulp_current_status | default('unknown') }}" + when: pulp_deployed + +- name: Check if Pulp is already at target version + ansible.builtin.set_fact: + pulp_upgrade_needed: "{{ pulp_target_image not in (pulp_current_image | default('')) }}" + when: pulp_deployed + +- name: Display skip message if already at target version + ansible.builtin.debug: + msg: "{{ upgrade_messages.pulp.already_upgraded }}" + when: + - pulp_deployed + - not (pulp_upgrade_needed | default(true)) + +- name: Check Pulp API status before upgrade + ansible.builtin.uri: + url: "{{ pulp_status_url }}" + method: GET + validate_certs: false + status_code: [200] + register: pulp_pre_health + retries: 3 + delay: 5 + until: pulp_pre_health.status == 200 + delegate_to: oim + delegate_facts: true + connection: ssh + failed_when: false + when: + - pulp_deployed + - pulp_upgrade_needed | default(true) + +- name: Display Pulp pre-upgrade health status + ansible.builtin.debug: + msg: "Pulp API status: {{ 'healthy (HTTP 200)' if pulp_pre_health.status | default(0) == 200 else 'not accessible (will attempt upgrade anyway)' }}" + when: + - pulp_deployed + - pulp_upgrade_needed | default(true) diff --git a/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml b/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml new file mode 100644 index 0000000000..c76460af9c --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/resolve_admin_ip.yml @@ -0,0 +1,95 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# ============================================================================ +# resolve_admin_ip.yml — Resolve Admin NIC IP for Pulp API Health Checks +# ============================================================================ +# Resolves the admin_nic_ip needed for Pulp API health check endpoints. +# +# Resolution order: +# 1. Primary: Extract from local_repo_access.yml (same as upgrade_k8s) +# 2. Fallback: Extract from network_spec.yml (same as prepare_oim) +# 3. Last resort: Use localhost +# +# Sets fact: +# admin_nic_ip — IP address for Pulp API health checks +# ============================================================================ + +- name: Resolve admin NIC IP for Pulp health checks + block: + # Primary method: Extract from local_repo_access.yml (same as upgrade_k8s) + - name: Check if local_repo_access.yml exists + ansible.builtin.stat: + path: "{{ local_repo_access_path }}" + register: local_repo_access_stat + + - name: Load local_repo_access.yml + ansible.builtin.slurp: + src: "{{ local_repo_access_path }}" + register: local_repo_access_raw + when: local_repo_access_stat.stat.exists + + - name: Parse local_repo_access.yml + ansible.builtin.set_fact: + _local_repo_access: "{{ local_repo_access_raw.content | b64decode | from_yaml }}" + when: local_repo_access_stat.stat.exists + + - name: Set admin_nic_ip from local_repo_access + ansible.builtin.set_fact: + admin_nic_ip: "{{ _local_repo_access.offline_tarball_path | regex_replace('^(https?)://([^:]+):.*', '\\2') }}" + when: + - local_repo_access_stat.stat.exists + - _local_repo_access.offline_tarball_path is defined + + # Fallback method: Extract from network_spec.yml + - name: Check if network_spec.yml exists + ansible.builtin.stat: + path: "{{ network_spec_path }}" + register: network_spec_stat + when: admin_nic_ip is not defined + + - name: Load network_spec.yml as fallback + ansible.builtin.include_vars: + file: "{{ network_spec_path }}" + when: + - admin_nic_ip is not defined + - network_spec_stat.stat.exists | default(false) + + # Networks is a list in network_spec.yml, parse it to extract admin_network + - name: Parse network_spec data into network_data + ansible.builtin.set_fact: + network_data: "{{ network_data | default({}) | combine({item.keys() | first: item.values() | first}) }}" + loop: "{{ Networks }}" + when: + - admin_nic_ip is not defined + - Networks is defined + + - name: Set admin_nic_ip from network_spec + ansible.builtin.set_fact: + admin_nic_ip: "{{ network_data.admin_network.primary_oim_admin_ip }}" + when: + - admin_nic_ip is not defined + - network_data is defined + - network_data.admin_network is defined + - network_data.admin_network.primary_oim_admin_ip is defined + + rescue: + - name: Fallback to localhost for admin_nic_ip + ansible.builtin.set_fact: + admin_nic_ip: "localhost" + +- name: Display resolved admin_nic_ip + ansible.builtin.debug: + msg: "Pulp health checks will use admin_nic_ip: {{ admin_nic_ip }}" diff --git a/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml new file mode 100644 index 0000000000..937558189f --- /dev/null +++ b/upgrade/roles/upgrade_pulp/tasks/upgrade_pulp_container.yml @@ -0,0 +1,108 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Upgrade Pulp Container (3.80 → 3.113) +# +# Steps: +# 1. Pull new Pulp image +# 2. Stop and remove old container +# 3. Update quadlet file with new image tag +# 4. Reload systemd and start container +# 5. Wait for initialization and run migrations +# =========================================================================== + +# --- 1. Pull new Pulp image --- +- name: Pull new Pulp image ({{ pulp_target_image }}) + ansible.builtin.command: podman pull {{ pulp_target_image }} + register: pulp_pull_result + retries: "{{ pull_image_retries }}" + delay: "{{ pull_image_delay }}" + until: pulp_pull_result.rc == 0 + changed_when: pulp_pull_result.rc == 0 + delegate_to: oim + delegate_facts: true + connection: ssh + +# --- 2. Stop and remove old container --- +- name: Stop Pulp service + ansible.builtin.systemd: + name: "{{ pulp_container_name }}" + state: stopped + delegate_to: oim + delegate_facts: true + connection: ssh + +- name: Remove old Pulp container + ansible.builtin.command: podman rm -f {{ pulp_container_name }} + changed_when: true + failed_when: false + delegate_to: oim + delegate_facts: true + connection: ssh + +# --- 3. Update quadlet file with new image --- +- name: Update Pulp image in quadlet file + ansible.builtin.lineinfile: + path: "{{ pulp_quadlet_path }}" + regexp: '^Image=.*' + line: "Image={{ pulp_target_image }}" + state: present + delegate_to: oim + delegate_facts: true + connection: ssh + +# --- 4. Reload systemd and start container --- +- name: Reload systemd daemon + ansible.builtin.systemd: + daemon_reload: true + delegate_to: oim + delegate_facts: true + connection: ssh + +- name: Start Pulp service + ansible.builtin.systemd: + name: "{{ pulp_container_name }}" + state: started + enabled: true + register: pulp_start_result + retries: "{{ pulp_startup_retries }}" + delay: "{{ pulp_startup_delay }}" + until: pulp_start_result is succeeded + delegate_to: oim + delegate_facts: true + connection: ssh + +# --- 5. Wait for initialization and run migrations --- +- name: Wait for Pulp services to initialize + ansible.builtin.pause: + seconds: "{{ pulp_init_wait }}" + +- name: Run Pulp database migrations + containers.podman.podman_container_exec: + name: "{{ pulp_container_name }}" + command: pulpcore-manager migrate --noinput + register: pulp_migrate_result + retries: 3 + delay: 10 + until: pulp_migrate_result.rc == 0 + delegate_to: oim + delegate_facts: true + connection: ssh + failed_when: false + +- name: Display migration result + ansible.builtin.debug: + msg: "Database migration: {{ 'completed' if pulp_migrate_result.rc | default(1) == 0 else 'skipped or not required' }}" diff --git a/upgrade/roles/upgrade_pulp/vars/main.yml b/upgrade/roles/upgrade_pulp/vars/main.yml new file mode 100644 index 0000000000..722272dde5 --- /dev/null +++ b/upgrade/roles/upgrade_pulp/vars/main.yml @@ -0,0 +1,105 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +# =========================================================================== +# Pulp Container Upgrade Configuration (3.80 → 3.113) +# =========================================================================== + +# File permissions +dir_permissions_755: "0755" +file_permissions_644: "0644" + +# Pulp container settings +pulp_container_name: "pulp" +pulp_target_tag: "3.113" +pulp_target_image: "docker.io/pulp/pulp:{{ pulp_target_tag }}" + +# Pulp quadlet file path +pulp_quadlet_path: "/etc/containers/systemd/{{ pulp_container_name }}.container" + +# OIM metadata (for resolving host-side shared storage paths) +oim_metadata_path: "/opt/omnia/.data/oim_metadata.yml" + +# Paths for resolving admin_nic_ip +local_repo_access_path: "/opt/omnia/provision/local_repo_access.yml" +network_spec_path: "{{ input_project_dir | default('/opt/omnia/input') }}/network_spec.yml" + +# Pulp API endpoint for health checks +pulp_protocol: "https" +pulp_port: "2225" +pulp_status_url: "{{ pulp_protocol }}://{{ admin_nic_ip | default('localhost') }}:{{ pulp_port }}/pulp/api/v3/status/" + +# Image pull settings +pull_image_retries: 5 +pull_image_delay: 10 + +# Pulp startup and health check settings +pulp_startup_retries: 10 +pulp_startup_delay: 10 +pulp_init_wait: 30 +pulp_health_retries: 12 +pulp_health_delay: 10 + +# Pulp shared storage paths (OIM container paths) +# These paths are inside the OIM container and mounted as volumes in the Pulp container +pulp_data_base_path: "/opt/omnia/pulp/settings" +pulp_pgsql_path: "{{ pulp_data_base_path }}/pgsql" + +# Backup settings +# PostgreSQL data must be backed up before upgrade because: +# - Pulp 3.80 uses PostgreSQL 12/13 +# - Pulp 3.113 uses PostgreSQL 16 +# - PostgreSQL cannot downgrade data files between major versions +pulp_backup_dir: "/opt/omnia/backups/upgrade/version_2.1.0.0/pulp" +pulp_pgsql_backup_path: "{{ pulp_backup_dir }}/pgsql" + +# Upgrade messages +upgrade_messages: + pulp: + not_deployed: | + Pulp container is not deployed on the OIM. Skipping Pulp upgrade. + If you need to deploy Pulp, run: ansible-playbook prepare_oim/prepare_oim.yml + already_upgraded: | + Pulp is already at target version ({{ pulp_target_image }}). Skipping upgrade. + upgrade_success: | + ════════════════════════════════════════════════════════════ + PULP UPGRADE COMPLETED SUCCESSFULLY + ════════════════════════════════════════════════════════════ + Previous image: {{ pulp_current_image | default('unknown') }} + New image: {{ pulp_target_image }} + Container: {{ pulp_container_name }} + Status: running + API endpoint: {{ pulp_status_url }} + ════════════════════════════════════════════════════════════ + upgrade_failure: | + ════════════════════════════════════════════════════════════ + PULP UPGRADE FAILED + ════════════════════════════════════════════════════════════ + Target image: {{ pulp_target_image }} + + Please check: + - Pulp container logs: podman logs {{ pulp_container_name }} + - Pulp service status: systemctl status {{ pulp_container_name }} + - Shared storage accessibility + - Network connectivity to container registry + ════════════════════════════════════════════════════════════ + pre_check_failure: | + Pulp pre-upgrade health check failed. + The Pulp API is not accessible at {{ pulp_status_url }}. + Please verify Pulp is running before attempting upgrade. + post_check_failure: | + Pulp post-upgrade health check failed. + The upgraded Pulp container may not be functioning correctly. + Check container logs: podman logs {{ pulp_container_name }} diff --git a/upgrade/upgrade.yml b/upgrade/upgrade.yml index 7bf744ca69..e2c27ad10b 100644 --- a/upgrade/upgrade.yml +++ b/upgrade/upgrade.yml @@ -453,7 +453,7 @@ ── Omnia Upgrade Execution Plan (in order) ────────────────── 1. oim → Upgrade OpenCHAMI control-plane containers - on the Omnia Infrastructure Manager + and Pulp container (3.80 → 3.113) on the OIM 2. build_stream → SKIPPED (not enabled in build_stream_config.yml) 3. local_repo → Synchronize Omnia 2.2 packages into the local Pulp repository for cluster nodes @@ -607,7 +607,7 @@ # Sub-flow imports (each sub-flow reads upgrade_manifest.yml and # skips if its component_status is already 'completed') # ────────────────────────────────────────────────────────────────────── -- name: Upgrade OIM tasks (includes OpenCHAMI) +- name: Upgrade OIM tasks (includes OpenCHAMI and Pulp) ansible.builtin.import_playbook: playbooks/upgrade_oim.yml tags: [oim] From c8b0504031ac4e3aedc19a3fdc07e1ed77d69e4b Mon Sep 17 00:00:00 2001 From: pullan1 Date: Tue, 30 Jun 2026 11:09:24 +0530 Subject: [PATCH 08/13] ansible lint fixes Signed-off-by: pullan1 --- rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml | 1 + rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml | 2 +- upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml | 1 + upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml b/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml index bc7d1dec53..826bb70bce 100644 --- a/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml +++ b/rollback/roles/rollback_pulp/tasks/cleanup_pulp_data.yml @@ -82,6 +82,7 @@ # ── Report disk usage ────────────────────────────────────────────────────── - name: Report disk usage after rollback ansible.builtin.shell: | + set -o pipefail echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)" echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)" echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)" diff --git a/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml index 59df22ca94..effbc75493 100644 --- a/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml +++ b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml @@ -73,7 +73,7 @@ # ── Check if already at rollback target version ─────────────────────── - name: Check if Pulp is already at rollback target version ansible.builtin.set_fact: - pulp_already_rolled_back: "{{ pulp_rollback_image in (pulp_current_image | default('')) }}" + pulp_already_rolled_back: "{{ pulp_rollback_image in ((pulp_current_image | default('')) | string) }}" when: pulp_deployed - name: Display skip message if already at target version diff --git a/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml b/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml index 0cac71372d..052227a6fd 100644 --- a/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml +++ b/upgrade/roles/upgrade_pulp/tasks/cleanup_after_upgrade.yml @@ -40,6 +40,7 @@ - name: Report disk usage after upgrade ansible.builtin.shell: | + set -o pipefail echo "Pulp storage: $(du -sh {{ oim_host_pulp_data_base }}/pulp_storage 2>/dev/null | cut -f1)" echo "PostgreSQL: $(du -sh {{ oim_host_pgsql_path }} 2>/dev/null | cut -f1)" echo "Backup: $(du -sh {{ oim_host_backup_dir }} 2>/dev/null | cut -f1)" diff --git a/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml index 5819991864..07708a2947 100644 --- a/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml +++ b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml @@ -64,7 +64,7 @@ - name: Check if Pulp is already at target version ansible.builtin.set_fact: - pulp_upgrade_needed: "{{ pulp_target_image not in (pulp_current_image | default('')) }}" + pulp_upgrade_needed: "{{ pulp_target_image not in ((pulp_current_image | default('')) | string) }}" when: pulp_deployed - name: Display skip message if already at target version From c83f9bc861db267cc565e61814f3668975e5e380 Mon Sep 17 00:00:00 2001 From: pullan1 Date: Tue, 30 Jun 2026 11:22:54 +0530 Subject: [PATCH 09/13] ansiblelint fixes Signed-off-by: pullan1 --- rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml | 2 +- upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml index effbc75493..71a5279528 100644 --- a/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml +++ b/rollback/roles/rollback_pulp/tasks/pre_rollback_checks.yml @@ -73,7 +73,7 @@ # ── Check if already at rollback target version ─────────────────────── - name: Check if Pulp is already at rollback target version ansible.builtin.set_fact: - pulp_already_rolled_back: "{{ pulp_rollback_image in ((pulp_current_image | default('')) | string) }}" + pulp_already_rolled_back: "{{ (pulp_rollback_image | default('')) in ((pulp_current_image | default('')) | string) }}" when: pulp_deployed - name: Display skip message if already at target version diff --git a/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml index 07708a2947..ca141b0396 100644 --- a/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml +++ b/upgrade/roles/upgrade_pulp/tasks/pre_upgrade_health_check.yml @@ -64,7 +64,7 @@ - name: Check if Pulp is already at target version ansible.builtin.set_fact: - pulp_upgrade_needed: "{{ pulp_target_image not in ((pulp_current_image | default('')) | string) }}" + pulp_upgrade_needed: "{{ (pulp_target_image | default('')) not in ((pulp_current_image | default('')) | string) }}" when: pulp_deployed - name: Display skip message if already at target version From 7b0dd17147e5b4d0d609c40c7aa53f968e1e560d Mon Sep 17 00:00:00 2001 From: Kratika Patidar Date: Tue, 30 Jun 2026 16:46:32 +0530 Subject: [PATCH 10/13] activemq image change (#4798) --- .../tasks/common/get_container_image_list.yml | 2 +- common/vars/image_vars.yml | 2 +- examples/catalog/catalog_rhel.json | 34 +++--- .../catalog_rhel_with_nfs_provisioner.json | 34 +++--- examples/catalog/catalog_rhel_x86_64.json | 34 +++--- .../x86_64/rhel/10.0/service_k8s_v1.35.1.json | 2 +- .../tasks/generate_telemetry_deployments.yml | 8 ++ .../idrac_telemetry/activemq-config.yaml.j2 | 101 ++++++++++++++++++ .../idrac_telemetry_statefulset.yaml.j2 | 8 ++ .../templates/telemetry/kustomization.yaml.j2 | 1 + provision/roles/telemetry/vars/main.yml | 2 +- 11 files changed, 173 insertions(+), 55 deletions(-) create mode 100644 provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 diff --git a/common/tasks/common/get_container_image_list.yml b/common/tasks/common/get_container_image_list.yml index 894f4110c6..21a333a2ef 100644 --- a/common/tasks/common/get_container_image_list.yml +++ b/common/tasks/common/get_container_image_list.yml @@ -24,7 +24,7 @@ container_images: >- {{ container_images + [ 'docker.io/library/mysql:' + mysql_tag, - 'docker.io/rmohr/activemq:' + activemq_tag, + 'docker.io/apache/activemq:' + activemq_tag, 'docker.io/library/golang:' + golang_tag, 'docker.io/prom/prometheus:' + prometheus_tag ] }} diff --git a/common/vars/image_vars.yml b/common/vars/image_vars.yml index 1915ac6673..97c07d9a75 100644 --- a/common/vars/image_vars.yml +++ b/common/vars/image_vars.yml @@ -19,7 +19,7 @@ squid_tag: "6.6-24.04_beta" pulp_tag: "3.113" mysql_tag: "9.3.0" prometheus_tag: "v3.4.1" -activemq_tag: "5.15.9" +activemq_tag: "5.19.7" grafana_image_tag: "12.0.1" loki_image_tag: "3.5.1" promtail_image_tag: "3.5.1" diff --git a/examples/catalog/catalog_rhel.json b/examples/catalog/catalog_rhel.json index fe8ac00cd1..062acaebcd 100644 --- a/examples/catalog/catalog_rhel.json +++ b/examples/catalog/catalog_rhel.json @@ -76,6 +76,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/calico/cni", "docker.io/calico/kube-controllers", "docker.io/calico/node", @@ -88,7 +89,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -155,6 +155,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/curlimages/curl", "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "docker.io/dellhpcomniaaisolution/kafkapump", @@ -164,7 +165,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -597,6 +597,21 @@ "Tag": "1.35.1", "Version": "1.35.1" }, + "docker.io/apache/activemq": { + "Name": "docker.io/apache/activemq", + "SupportedOS": [ + { + "Name": "RHEL", + "Version": "10.0" + } + ], + "Architecture": [ + "x86_64" + ], + "Type": "image", + "Tag": "5.19.7", + "Version": "5.19.7" + }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", "SupportedOS": [ @@ -777,21 +792,6 @@ "Tag": "1.29", "Version": "1.29" }, - "docker.io/rmohr/activemq": { - "Name": "docker.io/rmohr/activemq", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "5.15.9", - "Version": "5.15.9" - }, "docker.io/timberio/vector": { "Name": "docker.io/timberio/vector", "SupportedOS": [ diff --git a/examples/catalog/catalog_rhel_with_nfs_provisioner.json b/examples/catalog/catalog_rhel_with_nfs_provisioner.json index d11135023a..72af265baf 100644 --- a/examples/catalog/catalog_rhel_with_nfs_provisioner.json +++ b/examples/catalog/catalog_rhel_with_nfs_provisioner.json @@ -67,6 +67,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/calico/cni", "docker.io/calico/kube-controllers", "docker.io/calico/node", @@ -79,7 +80,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -146,6 +146,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/curlimages/curl", "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "docker.io/dellhpcomniaaisolution/kafkapump", @@ -155,7 +156,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -566,6 +566,21 @@ "Tag": "1.35.1", "Version": "1.35.1" }, + "docker.io/apache/activemq": { + "Name": "docker.io/apache/activemq", + "SupportedOS": [ + { + "Name": "RHEL", + "Version": "10.0" + } + ], + "Architecture": [ + "x86_64" + ], + "Type": "image", + "Tag": "5.19.7", + "Version": "5.19.7" + }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", "SupportedOS": [ @@ -746,21 +761,6 @@ "Tag": "1.29", "Version": "1.29" }, - "docker.io/rmohr/activemq": { - "Name": "docker.io/rmohr/activemq", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "5.15.9", - "Version": "5.15.9" - }, "docker.io/timberio/vector": { "Name": "docker.io/timberio/vector", "SupportedOS": [ diff --git a/examples/catalog/catalog_rhel_x86_64.json b/examples/catalog/catalog_rhel_x86_64.json index d0c97cbf25..33a8c6d8e6 100644 --- a/examples/catalog/catalog_rhel_x86_64.json +++ b/examples/catalog/catalog_rhel_x86_64.json @@ -77,6 +77,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/calico/cni", "docker.io/calico/kube-controllers", "docker.io/calico/node", @@ -89,7 +90,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -156,6 +156,7 @@ "device-mapper-multipath", "doca-ofed", "docker.io/alpine/kubectl", + "docker.io/apache/activemq", "docker.io/curlimages/curl", "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "docker.io/dellhpcomniaaisolution/kafkapump", @@ -165,7 +166,6 @@ "docker.io/library/mysql", "docker.io/library/python", "docker.io/nginxinc/nginx-unprivileged", - "docker.io/rmohr/activemq", "docker.io/timberio/vector", "docker.io/victoriametrics/operator", "docker.io/victoriametrics/operator_1", @@ -583,6 +583,21 @@ "Tag": "1.35.1", "Version": "1.35.1" }, + "docker.io/apache/activemq": { + "Name": "docker.io/apache/activemq", + "SupportedOS": [ + { + "Name": "RHEL", + "Version": "10.0" + } + ], + "Architecture": [ + "x86_64" + ], + "Type": "image", + "Tag": "5.19.7", + "Version": "5.19.7" + }, "docker.io/calico/cni": { "Name": "docker.io/calico/cni", "SupportedOS": [ @@ -763,21 +778,6 @@ "Tag": "1.29", "Version": "1.29" }, - "docker.io/rmohr/activemq": { - "Name": "docker.io/rmohr/activemq", - "SupportedOS": [ - { - "Name": "RHEL", - "Version": "10.0" - } - ], - "Architecture": [ - "x86_64" - ], - "Type": "image", - "Tag": "5.15.9", - "Version": "5.15.9" - }, "docker.io/timberio/vector": { "Name": "docker.io/timberio/vector", "SupportedOS": [ diff --git a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json index 8c18422fcc..9f91df50ad 100644 --- a/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json +++ b/input/config/x86_64/rhel/10.0/service_k8s_v1.35.1.json @@ -22,7 +22,7 @@ { "package": "docker.io/victoriametrics/vlagent", "type": "image", "tag": "v1.50.0" }, { "package": "docker.io/alpine/kubectl", "tag": "1.35.1", "type": "image" }, { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" }, - { "package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9" }, + { "package": "docker.io/apache/activemq", "type": "image", "tag": "5.19.7" }, { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" }, { "package": "docker.io/library/python", "type": "image", "tag": "3.12-slim" }, { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.3" }, diff --git a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml index 790e8231c6..93353ec50c 100644 --- a/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml +++ b/provision/roles/telemetry/tasks/generate_telemetry_deployments.yml @@ -171,6 +171,14 @@ when: item.skip_when is not defined or not item.skip_when | bool tags: telemetry_deployment +- name: Populate ActiveMQ ConfigMap + ansible.builtin.template: + src: 'telemetry/idrac_telemetry/activemq-config.yaml.j2' + dest: "{{ hostvars['localhost']['k8s_client_share_path'] }}/telemetry/deployments/activemq-config.yaml" + mode: "{{ hostvars['localhost']['file_permissions_644'] }}" + when: telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) | bool + tags: telemetry_deployment + - name: Populate iDRAC telemetry statefulset ansible.builtin.template: src: 'telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2' diff --git a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 new file mode 100644 index 0000000000..5538a366e6 --- /dev/null +++ b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/activemq-config.yaml.j2 @@ -0,0 +1,101 @@ +# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: activemq-config + namespace: {{ telemetry_namespace }} +data: + activemq.xml: | + + + + + + file:${activemq.conf}/credentials.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2 b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2 index 28d44cb0ea..e054cad6d1 100644 --- a/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/idrac_telemetry/idrac_telemetry_statefulset.yaml.j2 @@ -48,6 +48,9 @@ spec: spec: volumes: {% set types = telemetry_config.telemetry_sources.idrac.collection_targets | default([]) %} + - name: activemq-config + configMap: + name: activemq-config {% if 'kafka' in types %} # Mount Kafka cluster CA certificate for TLS verification - name: kafka-cluster-ca-cert @@ -139,6 +142,11 @@ spec: - name: activemq image: {{ activemq_image }} imagePullPolicy: IfNotPresent + volumeMounts: + - name: activemq-config + mountPath: /opt/apache-activemq/conf/activemq.xml + subPath: activemq.xml + readOnly: true resources: requests: cpu: {{ idrac_telemetry_resources.activemq.requests.cpu }} diff --git a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 index 4ff4d6e1db..8ef440a330 100644 --- a/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 +++ b/provision/roles/telemetry/templates/telemetry/kustomization.yaml.j2 @@ -54,6 +54,7 @@ resources: {% endif %} {% if telemetry_config.telemetry_sources.idrac.metrics_enabled | default(false) %} # iDRAC Telemetry Resources + - activemq-config.yaml - idrac_telemetry_statefulset.yaml - telemetry_cleaner_rbac.yaml - telemetry_pod_cleanup.yaml diff --git a/provision/roles/telemetry/vars/main.yml b/provision/roles/telemetry/vars/main.yml index 1337efb093..c527b33c79 100644 --- a/provision/roles/telemetry/vars/main.yml +++ b/provision/roles/telemetry/vars/main.yml @@ -53,7 +53,7 @@ idrac_telemetry_receiver_image: "{{ telemetry_images['dellhpcomniaaisolution/idr kafkapump_image: "{{ telemetry_images['dellhpcomniaaisolution/kafkapump'] | default('docker.io/dellhpcomniaaisolution/kafkapump:1.2') }}" victoriapump_image: "{{ telemetry_images['dellhpcomniaaisolution/victoriapump'] | default('docker.io/dellhpcomniaaisolution/victoriapump:1.2') }}" -activemq_image: "{{ telemetry_images['rmohr/activemq'] | default('docker.io/rmohr/activemq:5.15.9') }}" +activemq_image: "{{ telemetry_images['apache/activemq'] | default('docker.io/apache/activemq:5.19.7') }}" activemq_http_port_1: 8161 activemq_http_port_2: 61616 messagebus_http_port: 61613 From 4ffe501ba924e786eb0b4c79752355503cc355f2 Mon Sep 17 00:00:00 2001 From: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:32:23 +0530 Subject: [PATCH 11/13] Update fix_vip_split_brain.yml Signed-off-by: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> --- rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml b/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml index 3ba836ffb4..52dafd0031 100644 --- a/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml +++ b/rollback/roles/rollback_k8s/tasks/fix_vip_split_brain.yml @@ -18,6 +18,14 @@ # the split-brain and cleans up stale VIP addresses so only the # rightful lease holder owns the VIP. +# ── Set default _stale_vip_nodes if fix_vip already completed ───── +# This ensures _stale_vip_nodes variable is always defined, even when +# fix_vip stage is skipped because it's already completed. +- name: Set default _stale_vip_nodes when fix_vip already completed + ansible.builtin.set_fact: + _stale_vip_nodes: [] + when: (rollback_status.stages.fix_vip.status | default('pending')) == 'completed' + - name: Skip fix_vip if already completed ansible.builtin.debug: msg: "fix_vip already completed — skipping." From cb7d9c7408f7551a603ecc00d4829aa0833bfb8e Mon Sep 17 00:00:00 2001 From: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:34:11 +0530 Subject: [PATCH 12/13] Update stop_cluster.yml Signed-off-by: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> --- rollback/roles/rollback_k8s/tasks/stop_cluster.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rollback/roles/rollback_k8s/tasks/stop_cluster.yml b/rollback/roles/rollback_k8s/tasks/stop_cluster.yml index 2e3a75445b..7042848f51 100644 --- a/rollback/roles/rollback_k8s/tasks/stop_cluster.yml +++ b/rollback/roles/rollback_k8s/tasks/stop_cluster.yml @@ -18,6 +18,14 @@ # 2. Non-leader CPs # 3. VIP leader last (keeps API accessible as long as possible) +# ── Set default cp_shutdown_order if stop_cluster already completed ───── +# This ensures cp_shutdown_order variable is always defined, even when +# stop_cluster stage is skipped because it's already completed. +- name: Set default cp_shutdown_order when stop_cluster already completed + ansible.builtin.set_fact: + cp_shutdown_order: "{{ all_cp_nodes }}" + when: (rollback_status.stages.stop_cluster.status | default('pending')) == 'completed' + - name: Skip stop_cluster if already completed ansible.builtin.debug: msg: "stop_cluster already completed — skipping." From cf0672ff77427b910cae7a289481ad704db3ce14 Mon Sep 17 00:00:00 2001 From: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> Date: Wed, 1 Jul 2026 13:36:03 +0530 Subject: [PATCH 13/13] Update step_kubelet_restart.yml Signed-off-by: Katakam Rakesh Naga Sai <125246792+Katakam-Rakesh@users.noreply.github.com> --- .../upgrade_k8s/tasks/step_kubelet_restart.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml b/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml index 8cbea6d96a..55a1475410 100644 --- a/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml +++ b/upgrade/roles/upgrade_k8s/tasks/step_kubelet_restart.yml @@ -26,21 +26,17 @@ daemon_reload: true delegate_to: "{{ current_node_name }}" -# Part 4: Restart crio service -- name: Restart crio on {{ current_node_name }} - ansible.builtin.systemd: - name: crio - state: restarted - delegate_to: "{{ current_node_name }}" - -# Part 5: Restart kubelet service +# Part 4: Restart kubelet service +# Note: CRI-O was already restarted during the crio_install step. +# Restarting it again here is redundant and causes unnecessary pod disruption. +# Only kubelet needs to be restarted to apply the config.yaml and feature gate changes. - name: Restart kubelet on {{ current_node_name }} ansible.builtin.systemd: name: kubelet state: restarted delegate_to: "{{ current_node_name }}" -# Part 6: Wait for node to become Ready with correct version +# Part 5: Wait for node to become Ready with correct version - name: Wait for node to become Ready ansible.builtin.command: >- kubectl get node {{ node_ip }}