From 2504f65c5830bf6106a540da2552c82b6494302f Mon Sep 17 00:00:00 2001 From: John Lockman Date: Tue, 26 May 2026 14:22:36 -0500 Subject: [PATCH 01/13] feat: Add RKE2 as alternative Kubernetes distribution for service cluster Add support for deploying RKE2 (Rancher Kubernetes Engine 2) as an alternative to vanilla Kubernetes (kubeadm) for the service cluster. Users can choose the distribution via the new k8s_distro field in omnia_config.yml. Configuration changes: - Add k8s_distro field to service_k8s_cluster in omnia_config.yml (default: kubeadm) - Update omnia_config.json schema to validate k8s_distro enum (kubeadm/rke2) - Expand k8s_cni enum to include canal, cilium (RKE2-supported CNIs) Pipeline integration: - discovery.yml: Enable service_k8s tag when service_rke2 is in software_config.json - include_software_config.yml: Detect service_rke2 and set service_rke2_support fact - validate_software_config_json.yml: Handle service_rke2 arch and version detection - common_validation.py: Accept service_rke2 for cluster validation - image_package_collector.py: Dynamically select service_rke2.json for image builds - k8s_config/main.yml: Branch NFS setup based on k8s_distro - configure_cloud_init_group.yml: Select RKE2-specific templates when k8s_distro=rke2 New files: - 3 RKE2 cloud-init templates (first server, additional server, agent) - create_rke2_config_nfs.yml for RKE2-specific NFS directory setup - service_rke2.json package definitions for RHEL 10.0 x86_64 Key differences from kubeadm path: - RKE2 uses built-in containerd (no CRI-O) - RKE2 manages CNI lifecycle (calico/canal/cilium/flannel) - Token-based cluster join instead of kubeadm certificates - kube-vip deployed as RKE2 static pod manifest - RKE2 registries.yaml for Pulp mirror integration - Port 9345 for RKE2 supervisor API Existing kubeadm deployment is completely unaffected when k8s_distro=kubeadm (default). Signed-off-by: John Lockman --- .../input_validation/schema/omnia_config.json | 71 ++ .../validation_flows/common_validation.py | 20 +- .../modules/image_package_collector.py | 31 +- .../config/x86_64/rhel/10.0/service_rke2.json | 73 ++ input/omnia_config.yml | 19 + .../tasks/validate_software_config_json.yml | 14 +- provision/provision.yml | 3 + .../tasks/configure_cloud_init_group.yml | 12 +- ...ce_kube_control_plane_first_x86_64.yaml.j2 | 654 ++++++++++++++++++ ...-service_kube_control_plane_x86_64.yaml.j2 | 421 +++++++++++ .../ci-group-service_kube_node_x86_64.yaml.j2 | 267 +++++++ .../roles/configure_ochami/vars/main.yml | 1 + .../tasks/create_rke2_config_nfs.yml | 266 +++++++ provision/roles/k8s_config/tasks/main.yml | 17 +- provision/roles/k8s_config/vars/main.yml | 2 + .../tasks/include_software_config.yml | 17 +- 16 files changed, 1870 insertions(+), 18 deletions(-) create mode 100644 input/config/x86_64/rhel/10.0/service_rke2.json create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 create mode 100644 provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml diff --git a/common/library/module_utils/input_validation/schema/omnia_config.json b/common/library/module_utils/input_validation/schema/omnia_config.json index a985fc7d1e..17d01b4c2d 100644 --- a/common/library/module_utils/input_validation/schema/omnia_config.json +++ b/common/library/module_utils/input_validation/schema/omnia_config.json @@ -177,6 +177,77 @@ } ] } + }, + "service_rke2_k8s_cluster": { + "type": "array", + "description": "List of RKE2 Kubernetes cluster configurations.", + "items": { + "type": "object", + "properties": { + "cluster_name": { + "type": "string", + "minLength": 1, + "description": "Unique name for the RKE2 service cluster." + }, + "deployment": { + "type": "boolean" + }, + "k8s_cni": { + "enum": ["calico", "canal", "cilium", "flannel"], + "description": "K8s CNI plugin for the RKE2 cluster." + }, + "pod_external_ip_range": { + "description": "IP range for LoadBalancer external IPs.", + "type": "string", + "pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$" + }, + "k8s_service_addresses": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s service IPs." + }, + "k8s_pod_network_cidr": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s pod network." + }, + "nfs_storage_name": { + "type": "string", + "description": "The NFS client server name mentioned in storage_config.yml" + }, + "csi_powerscale_driver_secret_file_path": { + "description": "Absolute file path for the secret.yaml file.", + "type": "string", + "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$" + }, + "csi_powerscale_driver_values_file_path": { + "description": "File path for the values.yaml file.", + "type": "string", + "pattern": "^(|/?([a-zA-Z0-9._-]+/)*[a-zA-Z0-9._-]+\\.yaml)$" + } + }, + "required": [ + "cluster_name", + "k8s_cni", + "k8s_service_addresses" + ], + "allOf": [ + { + "if": { + "properties": { + "csi_powerscale_driver_secret_file_path": { + "type": "string", + "minLength": 1 + } + }, + "required": ["csi_powerscale_driver_secret_file_path"] + }, + "then": { + "required": ["csi_powerscale_driver_values_file_path"] + } + } + ] + } } }, "required": [ diff --git a/common/library/module_utils/input_validation/validation_flows/common_validation.py b/common/library/module_utils/input_validation/validation_flows/common_validation.py index de33d3a21f..ffbc5cc496 100644 --- a/common/library/module_utils/input_validation/validation_flows/common_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/common_validation.py @@ -214,8 +214,18 @@ def validate_software_config( ) ) - # Ensure ldms is not configured without service_k8s in softwares - if "ldms" in software_names and "service_k8s" not in software_names: + # Ensure service_k8s and service_rke2 are not both present + if "service_k8s" in software_names and "service_rke2" in software_names: + errors.append( + create_error_msg( + "Validation Error: ", + "service_k8s and service_rke2", + "cannot both be present in softwares. Use service_k8s for kubeadm or service_rke2 for RKE2, not both." + ) + ) + + # Ensure ldms is not configured without service_k8s or service_rke2 in softwares + if "ldms" in software_names and "service_k8s" not in software_names and "service_rke2" not in software_names: errors.append( create_error_msg( "Validation Error: ", @@ -234,7 +244,7 @@ def validate_software_config( ) # Check for required subgroups when specific software names are present - software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s"] + software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2"] for software_name in software_requiring_subgroups: if software_name in software_names: if software_name not in data or not data[software_name]: @@ -1463,6 +1473,10 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors, cluster_set["service_k8s_cluster"] = data.get( "service_k8s_cluster", []) + if "service_rke2" in softwares and "service_k8s" in tag_names: + cluster_set["service_rke2_k8s_cluster"] = data.get( + "service_rke2_k8s_cluster", []) + for k8s_cluster_type, k8s_clusters in cluster_set.items(): deployments_list = [k.get('deployment', False) for k in k8s_clusters] true_count = deployments_list.count(True) diff --git a/common/library/modules/image_package_collector.py b/common/library/modules/image_package_collector.py index 1f40fffbf7..6409b2b424 100644 --- a/common/library/modules/image_package_collector.py +++ b/common/library/modules/image_package_collector.py @@ -109,8 +109,11 @@ def collect_packages_from_json(sw_data, fg_name=None, elif service_k8s_defined: fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "") - if "service_k8s" in sw_data and "cluster" in sw_data["service_k8s"]: - for entry in sw_data["service_k8s"]["cluster"]: + # Determine the top-level key (service_k8s or service_rke2) + k8s_top_key = "service_rke2" if "service_rke2" in sw_data else "service_k8s" + + if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]: + for entry in sw_data[k8s_top_key]["cluster"]: if entry.get("type") == "rpm" and "package" in entry: packages.append(entry["package"]) @@ -175,8 +178,8 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir, sw_data, fg_name=fg_name, slurm_defined=True ) ) - elif json_file.startswith("service_k8s_v"): - # Handle versioned service_k8s_v.json files + elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json": + # Handle versioned service_k8s_v.json or service_rke2.json packages.extend( collect_packages_from_json( sw_data, fg_name=fg_name, service_k8s_defined=True @@ -256,12 +259,20 @@ def run_module(): # Only validate service_k8s version if k8s functional groups are present service_k8s_json = None - if needs_service_k8s: + if needs_service_k8s and "service_rke2" not in allowed_softwares: if not service_k8s_version: module.fail_json(msg="service_k8s version not found in software_config.json") service_k8s_json = f"service_k8s_v{service_k8s_version}.json" # pylint: disable=line-too-long + # Determine which K8s package JSON to use based on software_config.json + if "service_rke2" in allowed_softwares: + k8s_json = "service_rke2.json" + elif service_k8s_json: + k8s_json = service_k8s_json + else: + k8s_json = None + # Functional group → json files mapping software_map = { "os_x86_64": ["default_packages.json", "ldms.json"], @@ -280,12 +291,12 @@ def run_module(): ], } - # Add k8s functional groups to software_map only if service_k8s_json is available - if service_k8s_json: + # Add k8s functional groups to software_map if k8s json is available + if k8s_json: software_map.update({ - "service_kube_node_x86_64": [service_k8s_json], - "service_kube_control_plane_first_x86_64": [service_k8s_json], - "service_kube_control_plane_x86_64": [service_k8s_json], + "service_kube_node_x86_64": [k8s_json], + "service_kube_control_plane_first_x86_64": [k8s_json], + "service_kube_control_plane_x86_64": [k8s_json], }) compute_images_dict = {} diff --git a/input/config/x86_64/rhel/10.0/service_rke2.json b/input/config/x86_64/rhel/10.0/service_rke2.json new file mode 100644 index 0000000000..9b0a5fc7c7 --- /dev/null +++ b/input/config/x86_64/rhel/10.0/service_rke2.json @@ -0,0 +1,73 @@ +{ + "service_rke2": { + "cluster": [ + { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" }, + { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2" }, + { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" }, + { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" }, + { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/vminsert", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/victoriametrics/vmselect", "type": "image", "tag": "v1.128.0-cluster" }, + { "package": "docker.io/curlimages/curl", "type": "image", "tag": "8.17.0" }, + { "package": "docker.io/rmohr/activemq", "type": "image", "tag": "5.15.9" }, + { "package": "docker.io/library/mysql", "type": "image", "tag": "9.3.0" }, + { "package": "docker.io/dellhpcomniaaisolution/idrac_telemetry_receiver", "type": "image", "tag": "1.0" }, + { "package": "docker.io/dellhpcomniaaisolution/kafkapump", "type": "image", "tag": "1.0" }, + { "package": "docker.io/dellhpcomniaaisolution/victoriapump", "type": "image", "tag": "1.0" }, + { "package": "cryptography==45.0.7", "type": "pip_module" }, + { "package": "omsdk==1.2.518", "type": "pip_module" }, + { "package": "cffi==1.17.1", "type": "pip_module" }, + { "package": "quay.io/strimzi/operator", "tag": "0.48.0", "type": "image" }, + { "package": "quay.io/strimzi/kafka", "tag": "0.48.0-kafka-4.1.0", "type": "image" }, + { "package": "docker.io/dellhpcomniaaisolution/ubuntu-ldms", "tag": "1.0", "type": "image" }, + { "package": "strimzi-kafka-operator-helm-3-chart-0.48.0", "type": "tarball", "url": "https://github.com/strimzi/strimzi-kafka-operator/releases/download/0.48.0/strimzi-kafka-operator-helm-3-chart-0.48.0.tgz" }, + { "package": "quay.io/strimzi/kafka-bridge", "tag": "0.33.1", "type": "image" }, + { "package": "apptainer", "type": "rpm", "repo_name": "epel" }, + { "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"} + ] + }, + "service_kube_control_plane": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, + { "package": "PyMySQL==1.1.2", "type": "pip_module" } + ] + }, + "service_kube_control_plane_first": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { + "package": "metallb-native-v0.15.2", + "type": "manifest", + "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml" + }, + { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, + { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" }, + { "package": "PyMySQL==1.1.2", "type": "pip_module" } + ] + }, + + "service_kube_node": { + "cluster": [ + { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" } + ] + } +} diff --git a/input/omnia_config.yml b/input/omnia_config.yml index de54fddfd2..50ab4027c7 100644 --- a/input/omnia_config.yml +++ b/input/omnia_config.yml @@ -190,3 +190,22 @@ service_k8s_cluster: k8s_crio_storage_size: "20G" csi_powerscale_driver_secret_file_path: "" csi_powerscale_driver_values_file_path: "" + +# ----------------------------SERVICE RKE2 K8S-------------------------------------------------- +# To deploy RKE2 instead of vanilla Kubernetes (kubeadm), configure this block +# and add "service_rke2" to software_config.json softwares list instead of "service_k8s". +# RKE2 bundles its own container runtime (containerd) and manages the CNI lifecycle. +# k8s_cni supports "calico", "canal", "cilium", and "flannel". +# Only one of service_k8s_cluster or service_rke2_k8s_cluster should have deployment: true. +# Please ensure corresponding cluster entry is added to high_availability_config.yml. + +# service_rke2_k8s_cluster: +# - cluster_name: service_cluster +# deployment: false +# k8s_cni: "calico" +# pod_external_ip_range: "172.16.107.170-172.16.107.200" +# k8s_service_addresses: "10.233.0.0/18" +# k8s_pod_network_cidr: "10.233.64.0/18" +# nfs_storage_name: "nfs_k8s" +# csi_powerscale_driver_secret_file_path: "" +# csi_powerscale_driver_values_file_path: "" diff --git a/local_repo/roles/validation/tasks/validate_software_config_json.yml b/local_repo/roles/validation/tasks/validate_software_config_json.yml index 6e734957f1..e326d8481a 100644 --- a/local_repo/roles/validation/tasks/validate_software_config_json.yml +++ b/local_repo/roles/validation/tasks/validate_software_config_json.yml @@ -31,16 +31,27 @@ - name: Check if service k8s support is true ansible.builtin.set_fact: service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}" + service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" software_names: "{{ software_config.softwares | map(attribute='name') | select('defined') | list }}" - name: Build software JSON file list ansible.builtin.set_fact: software_json_list: "{{ software_names | map('regex_replace', '$', '.json') | list }}" +- name: Set service_k8s_support true when rke2 is selected + ansible.builtin.set_fact: + service_k8s_support: true + when: service_rke2_support | bool + - name: Get k8s archs ansible.builtin.set_fact: service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | first).arch | default(['x86_64']) }}" - when: service_k8s_support + when: service_k8s_support and not (service_rke2_support | bool) + +- name: Get rke2 archs + ansible.builtin.set_fact: + service_k8s_arch: "{{ (software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | first).arch | default(['x86_64']) }}" + when: service_rke2_support | bool - name: Get k8s archs ansible.builtin.set_fact: @@ -94,4 +105,5 @@ msg: "{{ fail_msg }}" when: - service_k8s_support + - not (service_rke2_support | default(false) | bool) - service_k8s_version not in supported_k8s_versions diff --git a/provision/provision.yml b/provision/provision.yml index 53868cfb68..18cf8cd629 100644 --- a/provision/provision.yml +++ b/provision/provision.yml @@ -67,6 +67,9 @@ ( ['service_k8s'] if ( 'service_k8s' in ( + lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') + | from_json).softwares | map(attribute='name') | list + or 'service_rke2' in ( lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') | from_json).softwares | map(attribute='name') | list ) else [] diff --git a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml index a999c8b9ef..22e5fd4b1c 100644 --- a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml +++ b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml @@ -26,11 +26,21 @@ failed_when: false when: not (hostvars['localhost']['upgrade_mode'] | default(false) | bool) +- name: Determine cloud-init template path based on k8s_distro # noqa: yaml[line-length] + ansible.builtin.set_fact: + ci_group_template_src: >- + {%- if k8s_distro | default('kubeadm') == 'rke2' and + 'service_kube' in functional_group_name -%} + cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2 + {%- else -%} + cloud_init/ci-group-{{ functional_group_name }}.yaml.j2 + {%- endif -%} + - name: Load ci group template block: - name: Load ci group template - {{ functional_group_name }} ansible.builtin.template: - src: "cloud_init/ci-group-{{ functional_group_name }}.yaml.j2" + src: "{{ ci_group_template_src | trim }}" dest: "{{ cloud_init_dir }}/ci-group-{{ functional_group_name }}.yaml" mode: "{{ hostvars['localhost']['file_permissions_644'] }}" rescue: diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 new file mode 100644 index 0000000000..6a913eee93 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -0,0 +1,654 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host {{ k8s_control_ssh_patterns }} + IdentityFile {{ k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/chrony.conf + permissions: '0644' + content: | + server {{ cluster_boot_ip }} iburst + + driftfile /var/lib/chrony/drift + rtcsync + makestep 1.0 3 + logdir /var/log/chrony + cmdport 0 + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + + - path: /etc/fstab + content: | + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/config.yaml + owner: root:root + permissions: '0600' + content: | + tls-san: + - {{ kube_vip }} + - {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + node-name: {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + write-kubeconfig-mode: "0644" + cluster-cidr: {{ k8s_pod_network_cidr }} + service-cidr: {{ k8s_service_addresses }} + cni: {{ k8s_cni }} + disable: + - rke2-ingress-nginx + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + HELM_VERSION="v3.19.0" + ARCH="amd64" + cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + + # Optional: Set up bash completion + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + - path: /tmp/ipaddress_pool.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: first-pool + namespace: metallb-system + spec: + addresses: + - {{ pod_external_ip_range }} + + - path: /tmp/l2advertisement.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: default + namespace: metallb-system + spec: + ipAddressPools: + - first-pool + +{% if hostvars['localhost']['idrac_telemetry_support'] or hostvars['localhost']['ldms_support'] %} + - path: /root/telemetry.sh + owner: root:root + permissions: '0755' + content: | + {{ lookup('template', 'templates/telemetry/telemetry.sh.j2') | indent(12) }} +{% endif %} + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports (Calico BGP, VXLAN, WireGuard, NodePort, MetalLB) + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + + # Enable services + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + + # Add pod/service networks + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + + # Set default zone to trusted + - firewall-cmd --set-default-zone=trusted + + # Reload the firewall rules + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/server/manifests + - | + tmpfile=$(mktemp) + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - mv /tmp/generate-control-plane-join.sh {{ k8s_client_mount_path }} || true + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.cluster_initialized" + export KUBECONFIG="/etc/rancher/rke2/rke2.yaml" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # FIRST BOOT - CLUSTER INIT + echo "Initial boot - initializing RKE2 first server for service_kube_control_plane_first_x86_64" + mv /tmp/ipaddress_pool.yaml {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + mv /tmp/l2advertisement.yaml {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml + + # Determine the VIP interface + NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') + sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml + + # Place kube-vip as a static pod for RKE2 + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml + + # Start RKE2 server + echo "Starting RKE2 server..." + systemctl enable rke2-server.service + systemctl start rke2-server.service + + # Wait for RKE2 to generate kubeconfig + echo "Waiting for RKE2 kubeconfig..." + while [ ! -f /etc/rancher/rke2/rke2.yaml ]; do + sleep 5 + done + + # Set up kubectl access + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + chmod 600 $HOME/.kube/config + + echo "Waiting for one Ready control plane node ..." + while true; do + control_plane_ready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) + if [ "$control_plane_ready" -ge 1 ]; then + echo "Found $control_plane_ready Ready node(s)!" + break + else + echo "No Ready node yet, waiting 5s ..." + sleep 5 + fi + done + + # Wait for all pods to be ready + echo "Waiting for all pods to be Ready (Running/Completed)..." + while true; do + not_ready=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '{ print $4 }' | grep -vE '^(Running|Completed)$' | wc -l) + if [ "${not_ready}" -eq 0 ]; then + echo "All pods are Running or Completed." + break + else + echo "$not_ready pods not yet ready, waiting 5s ..." + sleep 5 + fi + done + + # Save the server token to NFS for other nodes to join + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + while [ ! -f /var/lib/rancher/rke2/server/node-token ]; do + echo "Waiting for RKE2 node token..." + sleep 5 + done + cp /var/lib/rancher/rke2/server/node-token "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + chmod 644 "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + echo "RKE2 server token saved to NFS." + + # Update kubeconfig to use VIP + KUBE_VIP="{{ kube_vip }}" + KUBE_PORT="6443" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" /etc/rancher/rke2/rke2.yaml + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + + # Also save kubeconfig to NFS for other control planes + cp /etc/rancher/rke2/rke2.yaml "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + + echo "Waiting for at least one READY Kubernetes worker node ..." + while true; do + if kubectl get nodes --no-headers | grep -Ev 'control-plane|master' | grep ' Ready '; then + echo "Worker node(s) present and Ready." + break + else + echo "No Ready worker node detected yet. Retrying in 10 seconds..." + sleep 10 + fi + done + + echo "Installing plugins" + echo "Installing nfs-client-provisioner" + /usr/local/bin/helm install nfs-client {{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz \ + --namespace default --create-namespace \ + --set nfs.server={{ k8s_nfs_server_ip }} \ + --set nfs.path={{ k8s_server_share_path }} \ + --set storageClass.defaultClass=true \ + --set storageClass.reclaimPolicy=Retain + echo "Waiting for nfs-subdir-external-provisioner pods to appear..." + sleep 15 + + if kubectl get pods -n default -l app=nfs-subdir-external-provisioner | grep -q nfs-subdir; then + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + else + echo "Pods not yet created, retrying after 10 seconds..." + sleep 10 + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + fi + + echo "Installing MetalLB" + kubectl create -f {{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml + echo "Waiting for MetalLB pods to be ready..." + kubectl wait --namespace metallb-system --for=condition=Ready pods --all --timeout=300s + echo "Waiting for MetalLB webhook to be ready..." + until kubectl get endpoints metallb-webhook-service -n metallb-system \ + -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -qE '[0-9]'; do + echo "Webhook endpoints not ready yet. Retrying in 5s..." + sleep 5 + done + echo "MetalLB webhook is ready." + echo "Deploy ipaddress pool" + kubectl create -f {{ k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + echo "Deploy Layer2 Configuration" + kubectl create -f {{ k8s_client_mount_path }}/metallb/l2advertisement.yaml + + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + + CSI_DRIVER_SUPPORT="{{ csi_driver_powerscale_support | lower }}" + echo "===== Checking if PowerScale CSI driver support is enabled =====" + + if [ "$CSI_DRIVER_SUPPORT" != "true" ]; then + echo "PowerScale CSI driver support is disabled. Skipping deployment." + true + else + echo "PowerScale CSI driver support is enabled. Proceeding with deployment." + echo "===== Copying CSI PowerScale driver from NFS-mounted path =====" + mkdir -p /opt/omnia + POWERSCALE_DEPLOYMENT_FAILED=0 + + if cp -rp {{ k8s_client_mount_path }}/csi-driver-powerscale /opt/omnia/; then + echo "Copied CSI PowerScale driver to /opt/omnia successfully." + else + echo "ERROR: Failed to copy PowerScale driver. Skipping PowerScale deployment." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + SECRET_FILE="/opt/omnia/csi-driver-powerscale/secret.yaml" + echo "Checking if creds are provided by user" + + if [[ -f "$SECRET_FILE" ]]; then + echo "Found secret file at $SECRET_FILE" + csi_username=$(grep -v '^[[:space:]]*#' "$SECRET_FILE" | grep 'username:' | head -1 | awk -F':' '{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' | base64 --decode 2>/dev/null) + csi_password=$(grep -v '^[[:space:]]*#' "$SECRET_FILE" | grep 'password:' | head -1 | awk -F':' '{gsub(/^[[:space:]]+|[[:space:]]+$/, "", $2); print $2}' | base64 --decode 2>/dev/null) + + if [ -z "${csi_username}" ] || [ -z "${csi_password}" ]; then + echo " ERROR: CSI credentials not defined in secret.yaml." + POWERSCALE_DEPLOYMENT_FAILED=1 + else + export csi_username + export csi_password + fi + else + echo "ERROR: secret.yaml not found at $SECRET_FILE." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Checking if PowerScale driver is deployed =====" + if kubectl get pods -n isilon --no-headers 2>/dev/null | grep -q '^isilon-'; then + echo "PowerScale driver is already deployed on the cluster." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -ne 0 ]; then + echo "PowerScale prerequisites failed. Skipping remaining deployment steps." + true + else + echo "===== Ensuring 'isilon' namespace exists =====" + kubectl create namespace isilon --dry-run=client -o yaml | kubectl apply -f - || { + echo "ERROR: Failed to create or verify 'isilon' namespace." + POWERSCALE_DEPLOYMENT_FAILED=1 + } + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Creating and patching isilon-creds secret =====" + kubectl delete secret isilon-creds -n isilon >/dev/null 2>&1 || true + kubectl create secret generic isilon-creds -n isilon \ + --from-file=config="$SECRET_FILE" >/dev/null 2>&1 || POWERSCALE_DEPLOYMENT_FAILED=1 + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Applying empty certificate secret =====" + if [ -f "/opt/omnia/csi-driver-powerscale/empty_isilon-certs.yaml" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/empty_isilon-certs.yaml || { + echo "Failed to apply empty certs secret. Continuing..." + POWERSCALE_DEPLOYMENT_FAILED=1 + } + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Deploying External Snapshotter CRDs =====" + if [ -d "/opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/client/config/crd" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/client/config/crd/ >/dev/null 2>&1 || true + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Deploying Snapshot Controller =====" + if [ -d "/opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/deploy/kubernetes/snapshot-controller" ]; then + kubectl apply -f /opt/omnia/csi-driver-powerscale/csi-powerscale/external-snapshotter/deploy/kubernetes/snapshot-controller/ >/dev/null 2>&1 || true + kubectl set image deployment/snapshot-controller snapshot-controller=registry.k8s.io/sig-storage/snapshot-controller:v8.3.0 -n kube-system >/dev/null 2>&1 || true + kubectl rollout status deployment/snapshot-controller -n kube-system --timeout=300s >/dev/null 2>&1 || true + fi + fi + + if [ "$POWERSCALE_DEPLOYMENT_FAILED" -eq 0 ]; then + echo "===== Running CSI PowerScale installation script =====" + INSTALL_SCRIPT="/opt/omnia/csi-driver-powerscale/csi-powerscale/dell-csi-helm-installer/csi-install.sh" + if [ -x "$INSTALL_SCRIPT" ]; then + cd "$(dirname "$INSTALL_SCRIPT")" || true + ./csi-install.sh --namespace isilon --values /opt/omnia/csi-driver-powerscale/values.yaml & + CSI_PID=$! + wait $CSI_PID + else + echo "ERROR: CSI install script not found." + POWERSCALE_DEPLOYMENT_FAILED=1 + fi + fi + fi + fi + systemctl restart nfs-client.target + systemctl restart rpcbind + +{% if hostvars['localhost']['idrac_telemetry_support'] or hostvars['localhost']['ldms_support'] %} + echo "Applying Telemetry Kubernetes deployments" + /root/telemetry.sh +{% endif %} + + echo "Display nodes and pods status after deploying plugins" + echo "Listing all Kubernetes nodes:" + kubectl get nodes -o wide + + echo "Listing all Kubernetes pods in all namespaces:" + kubectl get pods --all-namespaces -o wide + + # Mark initialization complete so all of above is skipped on reboot! + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP INIT + echo "service_kube_control_plane_first_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 new file mode 100644 index 0000000000..3e30b9bc4c --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -0,0 +1,421 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host {{ k8s_control_ssh_patterns }} + IdentityFile {{ k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + HELM_VERSION="v3.19.0" + ARCH="amd64" + cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + + # Optional: Set up bash completion + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + + # Enable services + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + + # Add pod/service networks + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + + # Set default zone to trusted + - firewall-cmd --set-default-zone=trusted + + # Reload the firewall rules + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/server/manifests + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.cluster_initialized" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + set -e + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as additional server + echo "Initial boot - joining RKE2 cluster as additional server for service_kube_control_plane_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable." + + # Write RKE2 config for joining as server + cat > /etc/rancher/rke2/config.yaml </dev/null 2>&1; do + sleep 10 + done + kubectl get nodes -o wide + + systemctl restart nfs-client.target + systemctl restart rpcbind + # Mark initialization complete + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP JOIN + echo "service_kube_control_plane_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 new file mode 100644 index 0000000000..b3b163a9ea --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 @@ -0,0 +1,267 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host {{ k8s_control_ssh_patterns }} + IdentityFile {{ k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open required ports for RKE2 agent + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + + # Enable services + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + + # Add Kubernetes pod/service CIDRs + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} + + # Set default zone to trusted + - firewall-cmd --set-default-zone=trusted + + # Reload rules + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - systemctl restart rpcbind + - mount -a + - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - | + set -e + MARKER="/etc/rancher/rke2/.cluster_initialized" + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as agent (worker node) + echo "Initial boot - joining RKE2 cluster as agent for service_kube_node_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable. Joining this service_kube_node_x86_64 to the cluster now." + + # Write RKE2 agent config + cat > /etc/rancher/rke2/config.yaml < 0 + - item.item.dest_path | length > 0 + +- name: Copy aarch64 offline packages + ansible.builtin.copy: + src: "{{ item.item.source_path }}/" + dest: "{{ item.item.dest_path }}/" + remote_src: true + mode: preserve + loop: "{{ aarch64_offline_pkg_sources.results | default([]) }}" + when: + - item.stat.exists + - item.item.source_path | length > 0 + - item.item.dest_path | length > 0 + +- name: Include local repo access variable file + ansible.builtin.include_vars: "{{ local_repo_access_config_file }}" + +- name: Load service_rke2.json + ansible.builtin.set_fact: + k8s_packages_json: "{{ lookup('file', rke2_packages_file) | from_json }}" + +- name: Extract and set facts for tarball URLs (RKE2) # noqa: yaml[line-length] + vars: + cp_first_cluster: "{{ k8s_packages_json['service_kube_control_plane_first']['cluster'] }}" + ansible.builtin.set_fact: + metallb_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'manifest') + | selectattr('package', 'search', 'metallb-native') + | map(attribute='package') | join }} + helm_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'helm') + | map(attribute='package') | join }} + nfs_subdir_external_provisioner_pkg: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'nfs-subdir-external-provisioner') + | map(attribute='package') | join }} + +- name: Copy pulp webserver certificate to target host + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ anchors_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Update CA trust on target host + ansible.builtin.command: update-ca-trust + register: update_ca + changed_when: false + +# metallb +- name: Download metallb-native manifest + ansible.builtin.get_url: + url: "{{ metallb_manifest_yaml_url }}" + dest: "{{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml" + mode: "{{ file_mode }}" + +# helm +- name: Download helm tarball + ansible.builtin.get_url: + url: "{{ helm_tarball_url }}" + dest: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" + mode: "{{ file_mode }}" + +- name: Untar helm tarball repo + ansible.builtin.unarchive: + src: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" + dest: "{{ k8s_client_mount_path }}/helm/" + remote_src: true + +# nfs client provisioner +- name: Download nfs-client-provisioner tarball + ansible.builtin.get_url: + url: "{{ nfs_client_provisioner_tarball_url }}" + dest: "{{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz" + mode: "{{ file_mode }}" + +- name: Copy pulp webserver certificate to client_share_path + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ k8s_client_mount_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Include PowerScale CSI dependency tasks + ansible.builtin.include_tasks: get_powerscale_dependencies.yml + when: hostvars['localhost']['csi_driver_powerscale_support'] | bool diff --git a/provision/roles/k8s_config/tasks/main.yml b/provision/roles/k8s_config/tasks/main.yml index 2595c99080..5d7cb32686 100644 --- a/provision/roles/k8s_config/tasks/main.yml +++ b/provision/roles/k8s_config/tasks/main.yml @@ -13,6 +13,19 @@ # limitations under the License. --- -- name: Creating the configuration files required for service_k8s in nfs share - ansible.builtin.include_tasks: create_k8s_config_nfs.yml +- name: Set k8s_distro based on service_rke2_support + ansible.builtin.set_fact: + k8s_distro: "{{ 'rke2' if (hostvars['localhost']['service_rke2_support'] | default(false) | bool) else 'kubeadm' }}" when: hostvars['localhost']['service_k8s_support'] + +- name: Creating the configuration files required for kubeadm service_k8s in nfs share + ansible.builtin.include_tasks: create_k8s_config_nfs.yml + when: + - hostvars['localhost']['service_k8s_support'] + - k8s_distro | default('kubeadm') == 'kubeadm' + +- name: Creating the configuration files required for RKE2 service_k8s in nfs share + ansible.builtin.include_tasks: create_rke2_config_nfs.yml + when: + - hostvars['localhost']['service_k8s_support'] + - k8s_distro | default('kubeadm') == 'rke2' diff --git a/provision/roles/k8s_config/vars/main.yml b/provision/roles/k8s_config/vars/main.yml index c949564d70..3551f4da96 100644 --- a/provision/roles/k8s_config/vars/main.yml +++ b/provision/roles/k8s_config/vars/main.yml @@ -17,6 +17,7 @@ local_repo_access_config_file: "/opt/omnia/provision/local_repo_access.yml" input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}" # Versioned JSON file: service_k8s_v.json (e.g., service_k8s_v1.35.1.json) k8s_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_k8s_v{{ hostvars['localhost']['service_k8s_version'] }}.json" # noqa: yaml[line-length] +rke2_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_rke2.json" # noqa: yaml[line-length] calico_manifest_yaml_url: "{{ offline_manifest_path }}/{{ calico_package }}/{{ calico_package }}.yml" csi_powerscale_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/csi_driver_powerscale.json" # noqa: yaml[line-length] metallb_manifest_yaml_url: "{{ offline_manifest_path }}/{{ metallb_package }}/{{ metallb_package }}.yml" @@ -87,3 +88,4 @@ print_copy_msg: "Copying {{ item.name }} from {{ item.source_path }} to {{ item. offline_path_x86_64: [] offline_path_aarch64: [] ssh_private_key_path: /root/.ssh/oim_rsa +common_mode: "0755" diff --git a/provision/roles/provision_validations/tasks/include_software_config.yml b/provision/roles/provision_validations/tasks/include_software_config.yml index 2895762f8e..652d6620c0 100644 --- a/provision/roles/provision_validations/tasks/include_software_config.yml +++ b/provision/roles/provision_validations/tasks/include_software_config.yml @@ -53,15 +53,30 @@ - name: Initialise variables ansible.builtin.set_fact: service_k8s_support: false + service_rke2_support: false - name: Check if service k8s support is true ansible.builtin.set_fact: service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}" +- name: Check if service rke2 support is true + ansible.builtin.set_fact: + service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" + +- name: Set service_k8s_support true when rke2 is selected + ansible.builtin.set_fact: + service_k8s_support: true + when: service_rke2_support | bool + - name: Extract k8s version ansible.builtin.set_fact: service_k8s_version: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | map(attribute='version') | first }}" - when: service_k8s_support + when: service_k8s_support and not (service_rke2_support | bool) + +- name: Extract rke2 version + ansible.builtin.set_fact: + service_k8s_version: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | map(attribute='version') | first }}" + when: service_rke2_support | bool - name: Check if csi support is true ansible.builtin.set_fact: From d5a0a2350ef36f0949abb4d24278bf7721630e56 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Wed, 10 Jun 2026 19:49:58 +0000 Subject: [PATCH 02/13] fix: address RKE2 review findings for service cluster deployment - Read NFS share details from storage_config.yml 'mounts' (nfs_client_params no longer exists); use k8s_nfs_server_path in cloud-init fstab and nfs-client-provisioner values - Add rke2 (stable/1.35) and rke2-common RPM repos to local_repo_config.yml; source rke2-selinux from rke2-common - Set cluster-cidr/service-cidr on joining servers to avoid critical configuration value mismatch - Stage official RKE2 airgap image tarballs (core + CNI, v1.35.1+rke2r1) on the NFS share and import via /var/lib/rancher/rke2/agent/images so system images work offline - Deploy kube-vip as a kubelet static pod in agent/pod-manifests (not the server/manifests AddOn dir), mount kubeconfig at /etc/kubernetes/admin.conf, and place it after rke2.yaml exists to avoid hostPath dir race Signed-off-by: John Lockman --- .../config/x86_64/rhel/10.0/service_rke2.json | 7 +++- input/local_repo_config.yml | 7 +++- ...ce_kube_control_plane_first_x86_64.yaml.j2 | 25 +++++++++------ ...-service_kube_control_plane_x86_64.yaml.j2 | 23 +++++++++---- .../ci-group-service_kube_node_x86_64.yaml.j2 | 10 +++--- .../tasks/create_rke2_config_nfs.yml | 32 +++++++++++++++++-- 6 files changed, 78 insertions(+), 26 deletions(-) diff --git a/input/config/x86_64/rhel/10.0/service_rke2.json b/input/config/x86_64/rhel/10.0/service_rke2.json index 9b0a5fc7c7..d74d76cad7 100644 --- a/input/config/x86_64/rhel/10.0/service_rke2.json +++ b/input/config/x86_64/rhel/10.0/service_rke2.json @@ -9,7 +9,12 @@ { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, - { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2-common" }, + { "package": "rke2-images-core.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-core.linux-amd64.tar.zst" }, + { "package": "rke2-images-calico.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-calico.linux-amd64.tar.zst" }, + { "package": "rke2-images-canal.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-canal.linux-amd64.tar.zst" }, + { "package": "rke2-images-cilium.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-cilium.linux-amd64.tar.zst" }, + { "package": "rke2-images-flannel.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-flannel.linux-amd64.tar.zst" }, { "package": "docker.io/victoriametrics/victoria-metrics", "type": "image", "tag": "v1.128.0" }, { "package": "docker.io/victoriametrics/vmagent", "type": "image", "tag": "v1.128.0" }, { "package": "docker.io/victoriametrics/vmstorage", "type": "image", "tag": "v1.128.0-cluster" }, diff --git a/input/local_repo_config.yml b/input/local_repo_config.yml index 571c2947dd..e475e50198 100644 --- a/input/local_repo_config.yml +++ b/input/local_repo_config.yml @@ -67,7 +67,7 @@ # sslcacert : Path to SSL CA certificate (if using SSL) # sslclientkey: Path to SSL client key (if using SSL) # sslclientcert: Path to SSL client certificate (if using SSL) -# policy : Repository policy if mentioned allowed values (always, partial). +# policy : Repository policy if mentioned allowed values (always, partial). # If not provided, uses repo_config from software_config.json # caching : Enable or disable local caching. Allowed values: true, false (OPTIONAL) # If not provided, defaults to true @@ -194,11 +194,16 @@ rhel_subscription_repo_config_aarch64: # Version-specific naming is used only for service_k8s components (kubernetes, cri-o) # Other components (doca, cuda, slurm_custom) use non-versioned naming # ============================================================================ +# The 'rke2' repo channel (stable/1.35) must match the RKE2 minor version used in +# input/config/x86_64/rhel/10.0/service_rke2.json and the 'service_rke2' version +# in software_config.json. 'rke2-common' (noarch) provides rke2-selinux. omnia_repo_url_rhel_x86_64: - { url: "https://download.docker.com/linux/centos/10/x86_64/stable/", gpgkey: "https://download.docker.com/linux/centos/gpg", name: "docker-ce"} - { url: "https://dl.fedoraproject.org/pub/epel/10/Everything/x86_64/", gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-10", name: "epel"} - { url: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/", gpgkey: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "kubernetes-v1-35"} - { url: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/", gpgkey: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "cri-o-v1-35"} + - { url: "https://rpm.rancher.io/rke2/stable/1.35/centos/10/x86_64/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2"} + - { url: "https://rpm.rancher.io/rke2/stable/common/centos/10/noarch/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2-common"} - { url: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/", gpgkey: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/repodata/repomd.xml.key", name: "doca"} - { url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/", gpgkey: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/repodata/repomd.xml.key", name: "cuda"} - { url: "https://developer.download.nvidia.com/hpc-sdk/rhel/x86_64", gpgkey: "https://developer.download.nvidia.com/hpc-sdk/rhel/RPM-GPG-KEY-NVIDIA-HPC-SDK", name: "nvidia-hpc-sdk"} diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 index 6a913eee93..15c1540a9c 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -104,9 +104,9 @@ - path: /etc/fstab content: | - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 permissions: '0644' - path: /etc/rancher/rke2/config.yaml @@ -217,7 +217,7 @@ - NET_ADMIN - NET_RAW volumeMounts: - - mountPath: /etc/rancher/rke2/rke2.yaml + - mountPath: /etc/kubernetes/admin.conf name: kubeconfig hostAliases: - hostnames: @@ -338,7 +338,7 @@ - sudo modprobe nf_conntrack || true - sudo modprobe vxlan || true - sysctl --system - - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/server/manifests + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images - | tmpfile=$(mktemp) # Extract the first 'search' line only (ignore duplicates) @@ -360,6 +360,8 @@ chattr +i /etc/resolv.conf || true fi - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors - update-ca-trust extract - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf @@ -395,9 +397,6 @@ VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml - # Place kube-vip as a static pod for RKE2 - cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml - # Start RKE2 server echo "Starting RKE2 server..." systemctl enable rke2-server.service @@ -417,6 +416,12 @@ cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config chmod 600 $HOME/.kube/config + # Place kube-vip as a kubelet static pod now that rke2.yaml exists. + # /var/lib/rancher/rke2/agent/pod-manifests is RKE2's kubelet + # staticPodPath; deploying after the kubeconfig is written avoids + # the hostPath mount creating a directory at the rke2.yaml path. + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/agent/pod-manifests/kube-vip.yaml + echo "Waiting for one Ready control plane node ..." while true; do control_plane_ready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) @@ -477,8 +482,8 @@ echo "Installing nfs-client-provisioner" /usr/local/bin/helm install nfs-client {{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz \ --namespace default --create-namespace \ - --set nfs.server={{ k8s_nfs_server_ip }} \ - --set nfs.path={{ k8s_server_share_path }} \ + --set nfs.server={{ k8s_nfs_server_path.split(':')[0] }} \ + --set nfs.path={{ k8s_nfs_server_path.split(':')[1] }} \ --set storageClass.defaultClass=true \ --set storageClass.reclaimPolicy=Retain echo "Waiting for nfs-subdir-external-provisioner pods to appear..." diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 index 3e30b9bc4c..bc268a7951 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -91,9 +91,9 @@ permissions: '0644' - path: /etc/fstab content: | - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 permissions: '0644' - path: /etc/rancher/rke2/registries.yaml @@ -189,7 +189,7 @@ - NET_ADMIN - NET_RAW volumeMounts: - - mountPath: /etc/rancher/rke2/rke2.yaml + - mountPath: /etc/kubernetes/admin.conf name: kubeconfig hostAliases: - hostnames: @@ -276,7 +276,7 @@ - sudo modprobe nf_conntrack || true - sudo modprobe vxlan || true - sysctl --system - - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/server/manifests + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images - | tmpfile=$(mktemp) @@ -299,6 +299,8 @@ chattr +i /etc/resolv.conf || true fi - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors - update-ca-trust extract - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf @@ -358,16 +360,17 @@ - ${NODE_NAME} node-name: ${NODE_NAME} write-kubeconfig-mode: "0644" + cluster-cidr: {{ k8s_pod_network_cidr }} + service-cidr: {{ k8s_service_addresses }} cni: {{ k8s_cni }} disable: - rke2-ingress-nginx EOF - # Place kube-vip as a static pod for this server + # Prepare kube-vip manifest for this server (deployed after join) NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml - cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml # Start RKE2 server echo "Starting RKE2 server (joining cluster)..." @@ -387,6 +390,12 @@ cp -f /etc/rancher/rke2/rke2.yaml /root/.kube/config chmod 600 /root/.kube/config + # Place kube-vip as a kubelet static pod now that rke2.yaml exists. + # /var/lib/rancher/rke2/agent/pod-manifests is RKE2's kubelet + # staticPodPath; deploying after the kubeconfig is written avoids + # the hostPath mount creating a directory at the rke2.yaml path. + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/agent/pod-manifests/kube-vip.yaml + # Update kubeconfig to use VIP KUBE_PORT="6443" sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" /etc/rancher/rke2/rke2.yaml diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 index b3b163a9ea..11eeb37aaa 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_node_x86_64.yaml.j2 @@ -91,9 +91,9 @@ permissions: '0644' - path: /etc/fstab content: | - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 - {{ k8s_nfs_server_ip }}:{{ k8s_server_share_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }} {{ k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + {{ k8s_nfs_server_path }}/packages /var/lib/packages nfs noatime,nolock 0 0 permissions: '0644' - path: /etc/rancher/rke2/registries.yaml @@ -179,7 +179,7 @@ - sudo modprobe nf_conntrack || true - sudo modprobe vxlan || true - sysctl --system - - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 + - mkdir -p {{ k8s_client_mount_path }} /var/log/pods /var/lib/packages /etc/rancher/rke2 /var/lib/rancher/rke2/agent/images - | tmpfile=$(mktemp) @@ -203,6 +203,8 @@ fi - systemctl restart rpcbind - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ - cp {{ k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors - update-ca-trust extract - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf diff --git a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml index c8e604a496..170fe4e814 100644 --- a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml @@ -35,9 +35,9 @@ - name: Read the service_k8s mount point ansible.builtin.set_fact: - k8s_client_mount_path: "{{ (nfs_client_params | selectattr('nfs_name', 'equalto', nfs_storage_name) | first).client_share_path }}" - k8s_nfs_server_ip: "{{ (nfs_client_params | selectattr('nfs_name', 'equalto', nfs_storage_name) | first).server_ip }}" - k8s_server_share_path: "{{ (nfs_client_params | selectattr('nfs_name', 'equalto', nfs_storage_name) | first).server_share_path }}" + k8s_client_mount_path: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).mount_point }}" + k8s_nfs_server_ip: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).source.split(':')[0] }}" + k8s_nfs_server_path: "{{ (mounts | selectattr('name', 'equalto', nfs_storage_name) | first).source }}" - name: Ensure SSH key directory exists on K8s share ansible.builtin.file: @@ -98,6 +98,7 @@ - nfs-client-provisioner - helm - rke2 + - rke2/images - name: Create subdirectories on NFS share for CSI ansible.builtin.file: @@ -215,6 +216,19 @@ | selectattr('package', 'search', 'nfs-subdir-external-provisioner') | map(attribute='package') | join }} +- name: Extract RKE2 airgap image tarball package names # noqa: yaml[line-length] + vars: + rke2_cluster: "{{ k8s_packages_json['service_rke2']['cluster'] }}" + ansible.builtin.set_fact: + rke2_images_core_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-core') + | map(attribute='package') | join }} + rke2_images_cni_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-' ~ k8s_cni) + | map(attribute='package') | join }} + - name: Copy pulp webserver certificate to target host ansible.builtin.copy: src: "{{ pulp_webserver_cert_path }}" @@ -254,6 +268,18 @@ dest: "{{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz" mode: "{{ file_mode }}" +# RKE2 airgap system images (core + selected CNI) +# RKE2 auto-imports any *.tar.zst found in /var/lib/rancher/rke2/agent/images/ +# on the nodes; the cloud-init templates copy these from the NFS share. +- name: Download RKE2 airgap image tarballs to NFS share + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.gz" + dest: "{{ k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + mode: "{{ file_mode }}" + loop: + - "{{ rke2_images_core_pkg }}" + - "{{ rke2_images_cni_pkg }}" + - name: Copy pulp webserver certificate to client_share_path ansible.builtin.copy: src: "{{ pulp_webserver_cert_path }}" From 1b4969bd1c69dd6d26877adfb35fa5a0cc899b89 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Tue, 16 Jun 2026 09:46:03 -0500 Subject: [PATCH 03/13] fix: address code review findings for RKE2 service cluster feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix validation guard: use 'service_rke2' in tag_names (not 'service_k8s') when populating service_rke2_k8s_cluster for validation - Fix airgap image download: url used .tar.gz extension but files are .tar.zst - Add pod_external_ip_range and k8s_pod_network_cidr to required fields in omnia_config.json schema - Fix set -e placement: move to top of inline shell block in additional control plane cloud-init - Fix heredoc indentation: config.yaml content was over-indented by 2 spaces producing invalid YAML - Fix omnia_config.yml example: deployment should be true, not false - Move RKE2 repo comments inline with the two new repo entries in local_repo_config.yml - Add arch validation: error if service_rke2 is configured with non-x86_64 arch - Remove redundant inline comments across cloud-init templates and image_package_collector.py - Remove unused HELM_VERSION variable from install-helm.sh in both CP templates - Fix typo: 'nfs sever' → 'nfs server' in create_rke2_config_nfs.yml task name Signed-off-by: John Lockman Co-Authored-By: Claude Sonnet 4.6 --- .../input_validation/schema/omnia_config.json | 4 +- .../validation_flows/common_validation.py | 18 ++++++++- .../modules/image_package_collector.py | 2 - input/local_repo_config.yml | 5 +-- input/omnia_config.yml | 2 +- ...ce_kube_control_plane_first_x86_64.yaml.j2 | 12 ------ ...-service_kube_control_plane_x86_64.yaml.j2 | 38 +++++++------------ .../ci-group-service_kube_node_x86_64.yaml.j2 | 10 +---- .../tasks/create_rke2_config_nfs.yml | 4 +- 9 files changed, 39 insertions(+), 56 deletions(-) diff --git a/common/library/module_utils/input_validation/schema/omnia_config.json b/common/library/module_utils/input_validation/schema/omnia_config.json index 17d01b4c2d..d138b8e3b4 100644 --- a/common/library/module_utils/input_validation/schema/omnia_config.json +++ b/common/library/module_utils/input_validation/schema/omnia_config.json @@ -229,7 +229,9 @@ "required": [ "cluster_name", "k8s_cni", - "k8s_service_addresses" + "k8s_service_addresses", + "k8s_pod_network_cidr", + "pod_external_ip_range" ], "allOf": [ { diff --git a/common/library/module_utils/input_validation/validation_flows/common_validation.py b/common/library/module_utils/input_validation/validation_flows/common_validation.py index ffbc5cc496..9f06df74ec 100644 --- a/common/library/module_utils/input_validation/validation_flows/common_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/common_validation.py @@ -224,6 +224,22 @@ def validate_software_config( ) ) + # RKE2 packages are only available for x86_64; reject any other arch explicitly + if "service_rke2" in software_names: + rke2_entry = next( + (s for s in data.get("softwares", []) if s.get("name") == "service_rke2"), {} + ) + rke2_arch = rke2_entry.get("arch", ["x86_64"]) + unsupported_archs = [a for a in rke2_arch if a != "x86_64"] + if unsupported_archs: + errors.append( + create_error_msg( + "Validation Error: ", + "service_rke2", + f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}." + ) + ) + # Ensure ldms is not configured without service_k8s or service_rke2 in softwares if "ldms" in software_names and "service_k8s" not in software_names and "service_rke2" not in software_names: errors.append( @@ -1473,7 +1489,7 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors, cluster_set["service_k8s_cluster"] = data.get( "service_k8s_cluster", []) - if "service_rke2" in softwares and "service_k8s" in tag_names: + if "service_rke2" in softwares and "service_rke2" in tag_names: cluster_set["service_rke2_k8s_cluster"] = data.get( "service_rke2_k8s_cluster", []) diff --git a/common/library/modules/image_package_collector.py b/common/library/modules/image_package_collector.py index 6409b2b424..38f268e294 100644 --- a/common/library/modules/image_package_collector.py +++ b/common/library/modules/image_package_collector.py @@ -109,7 +109,6 @@ def collect_packages_from_json(sw_data, fg_name=None, elif service_k8s_defined: fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "") - # Determine the top-level key (service_k8s or service_rke2) k8s_top_key = "service_rke2" if "service_rke2" in sw_data else "service_k8s" if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]: @@ -291,7 +290,6 @@ def run_module(): ], } - # Add k8s functional groups to software_map if k8s json is available if k8s_json: software_map.update({ "service_kube_node_x86_64": [k8s_json], diff --git a/input/local_repo_config.yml b/input/local_repo_config.yml index e475e50198..dfe0834784 100644 --- a/input/local_repo_config.yml +++ b/input/local_repo_config.yml @@ -194,15 +194,14 @@ rhel_subscription_repo_config_aarch64: # Version-specific naming is used only for service_k8s components (kubernetes, cri-o) # Other components (doca, cuda, slurm_custom) use non-versioned naming # ============================================================================ -# The 'rke2' repo channel (stable/1.35) must match the RKE2 minor version used in -# input/config/x86_64/rhel/10.0/service_rke2.json and the 'service_rke2' version -# in software_config.json. 'rke2-common' (noarch) provides rke2-selinux. omnia_repo_url_rhel_x86_64: - { url: "https://download.docker.com/linux/centos/10/x86_64/stable/", gpgkey: "https://download.docker.com/linux/centos/gpg", name: "docker-ce"} - { url: "https://dl.fedoraproject.org/pub/epel/10/Everything/x86_64/", gpgkey: "https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-10", name: "epel"} - { url: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/", gpgkey: "https://pkgs.k8s.io/core:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "kubernetes-v1-35"} - { url: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/", gpgkey: "https://download.opensuse.org/repositories/isv:/cri-o:/stable:/v1.35/rpm/repodata/repomd.xml.key", name: "cri-o-v1-35"} + # Channel (stable/1.35) must match the minor version in service_rke2.json and software_config.json. - { url: "https://rpm.rancher.io/rke2/stable/1.35/centos/10/x86_64/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2"} + # noarch repo providing rke2-selinux - { url: "https://rpm.rancher.io/rke2/stable/common/centos/10/noarch/", gpgkey: "https://rpm.rancher.io/public.key", name: "rke2-common"} - { url: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/", gpgkey: "https://linux.mellanox.com/public/repo/doca/3.2.1/rhel10/x86_64/repodata/repomd.xml.key", name: "doca"} - { url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/", gpgkey: "https://developer.download.nvidia.com/compute/cuda/repos/rhel10/x86_64/repodata/repomd.xml.key", name: "cuda"} diff --git a/input/omnia_config.yml b/input/omnia_config.yml index 50ab4027c7..49bf3e3612 100644 --- a/input/omnia_config.yml +++ b/input/omnia_config.yml @@ -201,7 +201,7 @@ service_k8s_cluster: # service_rke2_k8s_cluster: # - cluster_name: service_cluster -# deployment: false +# deployment: true # k8s_cni: "calico" # pod_external_ip_range: "172.16.107.170-172.16.107.200" # k8s_service_addresses: "10.233.0.0/18" diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 index 15c1540a9c..0aee5ec615 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_first_x86_64.yaml.j2 @@ -236,12 +236,9 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.19.0" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm - - # Optional: Set up bash completion /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh chmod 0755 /etc/bash_completion.d/helm.sh @@ -318,19 +315,11 @@ - firewall-cmd --permanent --add-port=7946/udp - firewall-cmd --permanent --add-port=9090/tcp - firewall-cmd --permanent --add-port=8080/tcp - - # Enable services - firewall-cmd --permanent --add-service=http - firewall-cmd --permanent --add-service=https - - # Add pod/service networks - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} - - # Set default zone to trusted - firewall-cmd --set-default-zone=trusted - - # Reload the firewall rules - firewall-cmd --reload - sudo modprobe br_netfilter || true @@ -637,7 +626,6 @@ echo "Listing all Kubernetes pods in all namespaces:" kubectl get pods --all-namespaces -o wide - # Mark initialization complete so all of above is skipped on reboot! touch "$MARKER" echo "Cloud-Init has completed successfully." else diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 index bc268a7951..9233925c6d 100644 --- a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-service_kube_control_plane_x86_64.yaml.j2 @@ -208,12 +208,9 @@ content: | #!/bin/bash set -e - HELM_VERSION="v3.19.0" ARCH="amd64" cp {{ k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm chmod +x /usr/local/bin/helm - - # Optional: Set up bash completion /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh chmod 0755 /etc/bash_completion.d/helm.sh @@ -256,19 +253,11 @@ - firewall-cmd --permanent --add-port=7946/udp - firewall-cmd --permanent --add-port=9090/tcp - firewall-cmd --permanent --add-port=8080/tcp - - # Enable services - firewall-cmd --permanent --add-service=http - firewall-cmd --permanent --add-service=https - - # Add pod/service networks - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_service_addresses }} - firewall-cmd --permanent --zone=trusted --add-source={{ k8s_pod_network_cidr }} - - # Set default zone to trusted - firewall-cmd --set-default-zone=trusted - - # Reload the firewall rules - firewall-cmd --reload - sudo modprobe br_netfilter || true @@ -309,6 +298,7 @@ - /usr/local/bin/install-helm.sh - | + set -e echo "Installing Necessary Python pip packages" python3 -m ensurepip @@ -323,7 +313,6 @@ done MARKER="/etc/rancher/rke2/.cluster_initialized" export PATH="$PATH:/var/lib/rancher/rke2/bin" - set -e if [ ! -f "$MARKER" ]; then # Join RKE2 cluster as additional server echo "Initial boot - joining RKE2 cluster as additional server for service_kube_control_plane_x86_64" @@ -353,18 +342,18 @@ # Write RKE2 config for joining as server cat > /etc/rancher/rke2/config.yaml < Date: Mon, 22 Jun 2026 10:10:38 -0500 Subject: [PATCH 04/13] feat: add separate compute RKE2 cluster support Add compute_rke2 as an independent Kubernetes cluster type that can coexist with service_k8s (kubeadm), service_rke2, and slurm clusters. Configuration: - software_config.json: add compute_rke2 subgroups (compute_kube_*) - omnia_config.yml: add compute_rke2_k8s_cluster block (commented) - omnia_config.json: add compute_rke2_k8s_cluster schema - config.py: add compute_kube_* to FUNCTIONAL_GROUP_LAYER_MAP Packages: - compute_rke2.json: RKE2 core packages without service-layer images Validation: - common_validation.py: x86_64 arch check, subgroup requirement, NFS matching, cluster_set deployment validation for compute_rke2 - validate_software_config_json.yml: detect compute_rke2_support - include_software_config.yml: set compute_rke2_support fact Pipeline: - provision.yml: add compute_rke2 to omnia_run_tags - image_package_collector.py: add compute_kube_* functional groups Cloud-init: - 3 new templates: control_plane_first, control_plane, node - configure_cloud_init_group.yml: route compute_kube_* to rke2/ NFS: - create_compute_rke2_config_nfs.yml: compute cluster NFS setup - k8s_config/tasks/main.yml: include compute RKE2 NFS task Signed-off-by: John Lockman --- .../input_validation/common_utils/config.py | 9 + .../input_validation/schema/omnia_config.json | 47 ++ .../validation_flows/common_validation.py | 49 +- .../modules/image_package_collector.py | 27 +- .../config/x86_64/rhel/10.0/compute_rke2.json | 56 ++ input/omnia_config.yml | 18 + input/software_config.json | 8 + .../tasks/validate_software_config_json.yml | 1 + provision/provision.yml | 7 + .../tasks/configure_cloud_init_group.yml | 2 + ...te_kube_control_plane_first_x86_64.yaml.j2 | 523 ++++++++++++++++++ ...-compute_kube_control_plane_x86_64.yaml.j2 | 414 ++++++++++++++ .../ci-group-compute_kube_node_x86_64.yaml.j2 | 260 +++++++++ .../roles/configure_ochami/vars/main.yml | 13 + .../tasks/create_compute_rke2_config_nfs.yml | 197 +++++++ provision/roles/k8s_config/tasks/main.yml | 5 + provision/roles/k8s_config/vars/main.yml | 1 + .../tasks/include_software_config.yml | 5 + 18 files changed, 1626 insertions(+), 16 deletions(-) create mode 100644 input/config/x86_64/rhel/10.0/compute_rke2.json create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 create mode 100644 provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 create mode 100644 provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml diff --git a/common/library/module_utils/input_validation/common_utils/config.py b/common/library/module_utils/input_validation/common_utils/config.py index 588d112082..8037284c6d 100644 --- a/common/library/module_utils/input_validation/common_utils/config.py +++ b/common/library/module_utils/input_validation/common_utils/config.py @@ -38,6 +38,7 @@ "x86_64": [ "slurm_control_node", "slurm_node", "login_node", "login_compiler_node", "service_kube_control_plane", "service_kube_control_plane_first", "service_kube_node", + "compute_kube_control_plane", "compute_kube_control_plane_first", "compute_kube_node", "os" ], "aarch64": [ @@ -96,6 +97,11 @@ files["storage_config"], files["high_availability_config"], ], + "compute_rke2": [ + files["omnia_config"], + files["storage_config"], + files["high_availability_config"], + ], "storage": [files["storage_config"]], "prepare_oim": [ files["network_spec"], @@ -191,6 +197,9 @@ "service_kube_control_plane_first_x86_64": "management", "service_kube_control_plane_x86_64": "management", "service_kube_node_x86_64": "management", + "compute_kube_control_plane_first_x86_64": "compute", + "compute_kube_control_plane_x86_64": "compute", + "compute_kube_node_x86_64": "compute", "login_node_x86_64": "management", "login_node_aarch64": "management", "login_compiler_node_x86_64": "management", diff --git a/common/library/module_utils/input_validation/schema/omnia_config.json b/common/library/module_utils/input_validation/schema/omnia_config.json index d138b8e3b4..1ae1459583 100644 --- a/common/library/module_utils/input_validation/schema/omnia_config.json +++ b/common/library/module_utils/input_validation/schema/omnia_config.json @@ -250,6 +250,53 @@ } ] } + }, + "compute_rke2_k8s_cluster": { + "type": "array", + "description": "List of RKE2-based compute Kubernetes cluster configurations.", + "items": { + "type": "object", + "properties": { + "cluster_name": { + "type": "string", + "minLength": 1, + "description": "Unique name for the RKE2 compute cluster." + }, + "deployment": { + "type": "boolean" + }, + "k8s_cni": { + "enum": ["calico", "canal", "cilium", "flannel"], + "description": "K8s CNI plugin for the compute RKE2 cluster." + }, + "pod_external_ip_range": { + "description": "IP range for LoadBalancer external IPs.", + "type": "string", + "pattern": "^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)-(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))|(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$|^$" + }, + "k8s_service_addresses": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s service IPs." + }, + "k8s_pod_network_cidr": { + "type": "string", + "pattern": "^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/[0-9]{1,2}$", + "description": "CIDR for K8s pod network." + }, + "nfs_storage_name": { + "type": "string", + "description": "The NFS client server name mentioned in storage_config.yml" + } + }, + "required": [ + "cluster_name", + "k8s_cni", + "k8s_service_addresses", + "k8s_pod_network_cidr", + "pod_external_ip_range" + ] + } } }, "required": [ diff --git a/common/library/module_utils/input_validation/validation_flows/common_validation.py b/common/library/module_utils/input_validation/validation_flows/common_validation.py index 9f06df74ec..52c6ce9e0c 100644 --- a/common/library/module_utils/input_validation/validation_flows/common_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/common_validation.py @@ -225,20 +225,21 @@ def validate_software_config( ) # RKE2 packages are only available for x86_64; reject any other arch explicitly - if "service_rke2" in software_names: - rke2_entry = next( - (s for s in data.get("softwares", []) if s.get("name") == "service_rke2"), {} - ) - rke2_arch = rke2_entry.get("arch", ["x86_64"]) - unsupported_archs = [a for a in rke2_arch if a != "x86_64"] - if unsupported_archs: - errors.append( - create_error_msg( - "Validation Error: ", - "service_rke2", - f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}." - ) + for rke2_sw_name in ["service_rke2", "compute_rke2"]: + if rke2_sw_name in software_names: + rke2_entry = next( + (s for s in data.get("softwares", []) if s.get("name") == rke2_sw_name), {} ) + rke2_arch = rke2_entry.get("arch", ["x86_64"]) + unsupported_archs = [a for a in rke2_arch if a != "x86_64"] + if unsupported_archs: + errors.append( + create_error_msg( + "Validation Error: ", + rke2_sw_name, + f"is only supported on x86_64. Unsupported arch(es): {', '.join(unsupported_archs)}." + ) + ) # Ensure ldms is not configured without service_k8s or service_rke2 in softwares if "ldms" in software_names and "service_k8s" not in software_names and "service_rke2" not in software_names: @@ -260,7 +261,7 @@ def validate_software_config( ) # Check for required subgroups when specific software names are present - software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2"] + software_requiring_subgroups = ["additional_packages", "slurm_custom", "service_k8s", "service_rke2", "compute_rke2"] for software_name in software_requiring_subgroups: if software_name in software_names: if software_name not in data or not data[software_name]: @@ -579,6 +580,22 @@ def get_matching_clusters_for_nfs(nfs_name, omnia_config): ): matching_clusters["service_k8s_cluster"] = svc + # Service RKE2 + for svc in omnia_config.get("service_rke2_k8s_cluster", []): + if ( + svc.get("nfs_storage_name") == nfs_name + and svc.get("deployment") is True + ): + matching_clusters["service_rke2_k8s_cluster"] = svc + + # Compute RKE2 + for compute in omnia_config.get("compute_rke2_k8s_cluster", []): + if ( + compute.get("nfs_storage_name") == nfs_name + and compute.get("deployment") is True + ): + matching_clusters["compute_rke2_k8s_cluster"] = compute + # Slurm for slurm in omnia_config.get("slurm_cluster", []): if slurm.get("nfs_storage_name") == nfs_name: @@ -1493,6 +1510,10 @@ def validate_k8s(data, admin_networks, softwares, ha_config, tag_names, errors, cluster_set["service_rke2_k8s_cluster"] = data.get( "service_rke2_k8s_cluster", []) + if "compute_rke2" in softwares and "compute_rke2" in tag_names: + cluster_set["compute_rke2_k8s_cluster"] = data.get( + "compute_rke2_k8s_cluster", []) + for k8s_cluster_type, k8s_clusters in cluster_set.items(): deployments_list = [k.get('deployment', False) for k in k8s_clusters] true_count = deployments_list.count(True) diff --git a/common/library/modules/image_package_collector.py b/common/library/modules/image_package_collector.py index 38f268e294..9904bdbcaf 100644 --- a/common/library/modules/image_package_collector.py +++ b/common/library/modules/image_package_collector.py @@ -109,7 +109,11 @@ def collect_packages_from_json(sw_data, fg_name=None, elif service_k8s_defined: fg_name = fg_name.replace("_aarch64", "").replace("_x86_64", "") - k8s_top_key = "service_rke2" if "service_rke2" in sw_data else "service_k8s" + k8s_top_key = ( + "compute_rke2" if "compute_rke2" in sw_data + else "service_rke2" if "service_rke2" in sw_data + else "service_k8s" + ) if k8s_top_key in sw_data and "cluster" in sw_data[k8s_top_key]: for entry in sw_data[k8s_top_key]["cluster"]: @@ -159,6 +163,7 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir, # Remove version suffix for versioned files (e.g., service_k8s_v1.35.1 -> service_k8s) if sw_name.startswith("service_k8s_v"): sw_name = "service_k8s" + # compute_rke2.json -> compute_rke2 (already handled by replace above) if sw_name not in allowed_softwares: continue @@ -177,7 +182,7 @@ def process_functional_group(fg_name, arch, os_version, input_project_dir, sw_data, fg_name=fg_name, slurm_defined=True ) ) - elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json": + elif json_file.startswith("service_k8s_v") or json_file == "service_rke2.json" or json_file == "compute_rke2.json": # Handle versioned service_k8s_v.json or service_rke2.json packages.extend( collect_packages_from_json( @@ -253,8 +258,16 @@ def run_module(): "service_kube_control_plane_x86_64" } + # Compute RKE2 functional groups + compute_k8s_functional_groups = { + "compute_kube_node_x86_64", + "compute_kube_control_plane_first_x86_64", + "compute_kube_control_plane_x86_64" + } + # Check if any k8s functional groups are being processed needs_service_k8s = any(fg in k8s_functional_groups for fg in functional_groups) + needs_compute_rke2 = any(fg in compute_k8s_functional_groups for fg in functional_groups) # Only validate service_k8s version if k8s functional groups are present service_k8s_json = None @@ -272,6 +285,9 @@ def run_module(): else: k8s_json = None + # Determine compute RKE2 package JSON + compute_rke2_json = "compute_rke2.json" if "compute_rke2" in allowed_softwares else None + # Functional group → json files mapping software_map = { "os_x86_64": ["default_packages.json", "ldms.json"], @@ -297,6 +313,13 @@ def run_module(): "service_kube_control_plane_x86_64": [k8s_json], }) + if compute_rke2_json: + software_map.update({ + "compute_kube_node_x86_64": [compute_rke2_json], + "compute_kube_control_plane_first_x86_64": [compute_rke2_json], + "compute_kube_control_plane_x86_64": [compute_rke2_json], + }) + compute_images_dict = {} for fg_name in functional_groups: diff --git a/input/config/x86_64/rhel/10.0/compute_rke2.json b/input/config/x86_64/rhel/10.0/compute_rke2.json new file mode 100644 index 0000000000..263b728b5b --- /dev/null +++ b/input/config/x86_64/rhel/10.0/compute_rke2.json @@ -0,0 +1,56 @@ +{ + "compute_rke2": { + "cluster": [ + { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" }, + { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, + { "package": "rke2-selinux", "type": "rpm", "repo_name": "rke2-common" }, + { "package": "rke2-images-core.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-core.linux-amd64.tar.zst" }, + { "package": "rke2-images-calico.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-calico.linux-amd64.tar.zst" }, + { "package": "rke2-images-canal.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-canal.linux-amd64.tar.zst" }, + { "package": "rke2-images-cilium.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-cilium.linux-amd64.tar.zst" }, + { "package": "rke2-images-flannel.linux-amd64", "type": "tarball", "url": "https://github.com/rancher/rke2/releases/download/v1.35.1+rke2r1/rke2-images-flannel.linux-amd64.tar.zst" }, + { "package": "apptainer", "type": "rpm", "repo_name": "epel" }, + { "package": "doca-ofed", "type": "iso", "url": "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.1/host/doca-host-3.2.1-044000_25.10_rhel10.x86_64.rpm"} + ] + }, + "compute_kube_control_plane": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" } + ] + }, + "compute_kube_control_plane_first": { + "cluster": [ + { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { + "package": "metallb-native-v0.15.2", + "type": "manifest", + "url": "https://raw.githubusercontent.com/metallb/metallb/v0.15.2/config/manifests/metallb-native.yaml" + }, + { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, + { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, + { "package": "prettytable==3.14.0", "type": "pip_module" }, + { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, + { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "kubernetes==33.1.0", "type": "pip_module" } + ] + }, + "compute_kube_node": { + "cluster": [ + { "package": "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner", "tag": "v4.0.2", "type": "image" }, + { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, + { "package": "quay.io/metallb/controller", "tag": "v0.15.2", "type": "image" } + ] + } +} diff --git a/input/omnia_config.yml b/input/omnia_config.yml index 49bf3e3612..032a01e274 100644 --- a/input/omnia_config.yml +++ b/input/omnia_config.yml @@ -209,3 +209,21 @@ service_k8s_cluster: # nfs_storage_name: "nfs_k8s" # csi_powerscale_driver_secret_file_path: "" # csi_powerscale_driver_values_file_path: "" + +# ----------------------------COMPUTE RKE2 K8S------------------------------------------------- +# To deploy a separate RKE2-based compute Kubernetes cluster, configure this block +# and add "compute_rke2" to software_config.json softwares list. +# The compute RKE2 cluster is independent of the service cluster and can coexist +# with service_k8s (kubeadm), service_rke2, and slurm clusters. +# k8s_cni supports "calico", "canal", "cilium", and "flannel". +# Please ensure a corresponding cluster entry is added to high_availability_config.yml. +# CIDRs must not overlap with the service cluster CIDRs. + +# compute_rke2_k8s_cluster: +# - cluster_name: compute_cluster +# deployment: false +# k8s_cni: "calico" +# pod_external_ip_range: "" +# k8s_service_addresses: "10.234.0.0/18" +# k8s_pod_network_cidr: "10.234.64.0/18" +# nfs_storage_name: "nfs_compute_k8s" diff --git a/input/software_config.json b/input/software_config.json index 1d43f57024..457b0a0348 100644 --- a/input/software_config.json +++ b/input/software_config.json @@ -23,10 +23,18 @@ {"name": "service_kube_control_plane"}, {"name": "service_kube_node"} ], + "compute_rke2": [ + {"name": "compute_kube_control_plane_first"}, + {"name": "compute_kube_control_plane"}, + {"name": "compute_kube_node"} + ], "additional_packages":[ {"name": "service_kube_control_plane_first"}, {"name": "service_kube_control_plane"}, {"name": "service_kube_node"}, + {"name": "compute_kube_control_plane_first"}, + {"name": "compute_kube_control_plane"}, + {"name": "compute_kube_node"}, {"name": "slurm_control_node"}, {"name": "slurm_node"}, {"name": "login_node"}, diff --git a/local_repo/roles/validation/tasks/validate_software_config_json.yml b/local_repo/roles/validation/tasks/validate_software_config_json.yml index e326d8481a..77229390e8 100644 --- a/local_repo/roles/validation/tasks/validate_software_config_json.yml +++ b/local_repo/roles/validation/tasks/validate_software_config_json.yml @@ -32,6 +32,7 @@ ansible.builtin.set_fact: service_k8s_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_k8s') | list | length > 0 }}" service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" + compute_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'compute_rke2') | list | length > 0 }}" software_names: "{{ software_config.softwares | map(attribute='name') | select('defined') | list }}" - name: Build software JSON file list diff --git a/provision/provision.yml b/provision/provision.yml index 18cf8cd629..5bf52d166e 100644 --- a/provision/provision.yml +++ b/provision/provision.yml @@ -73,6 +73,13 @@ lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') | from_json).softwares | map(attribute='name') | list ) else [] + ) + + ( + ['compute_rke2'] + if 'compute_rke2' in ( + lookup('file', hostvars['localhost']['input_project_dir'] ~ '/software_config.json') + | from_json).softwares | map(attribute='name') | list + else [] ) ) | unique }} diff --git a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml index 22e5fd4b1c..1ac7a24fdf 100644 --- a/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml +++ b/provision/roles/configure_ochami/tasks/configure_cloud_init_group.yml @@ -32,6 +32,8 @@ {%- if k8s_distro | default('kubeadm') == 'rke2' and 'service_kube' in functional_group_name -%} cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2 + {%- elif 'compute_kube' in functional_group_name -%} + cloud_init/rke2/ci-group-{{ functional_group_name }}.yaml.j2 {%- else -%} cloud_init/ci-group-{{ functional_group_name }}.yaml.j2 {%- endif -%} diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 new file mode 100644 index 0000000000..7e978e74af --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_first_x86_64.yaml.j2 @@ -0,0 +1,523 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/chrony.conf + permissions: '0644' + content: | + server {{ cluster_boot_ip }} iburst + + driftfile /var/lib/chrony/drift + rtcsync + makestep 1.0 3 + logdir /var/log/chrony + cmdport 0 + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/config.yaml + owner: root:root + permissions: '0600' + content: | + tls-san: + - {{ compute_kube_vip }} + - {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + node-name: {% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %} + write-kubeconfig-mode: "0644" + cluster-cidr: {{ compute_k8s_pod_network_cidr }} + service-cidr: {{ compute_k8s_service_addresses }} + cni: {{ compute_k8s_cni }} + disable: + - rke2-ingress-nginx + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ compute_kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + ARCH="amd64" + cp {{ compute_k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + - path: /tmp/ipaddress_pool.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: first-pool + namespace: metallb-system + spec: + addresses: + - {{ compute_pod_external_ip_range }} + + - path: /tmp/l2advertisement.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + metadata: + name: default + namespace: metallb-system + spec: + ipAddressPools: + - first-pool + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports (Calico BGP, VXLAN, WireGuard, NodePort, MetalLB) + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - mv /tmp/generate-control-plane-join.sh {{ compute_k8s_client_mount_path }} || true + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + export KUBECONFIG="/etc/rancher/rke2/rke2.yaml" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # FIRST BOOT - CLUSTER INIT + echo "Initial boot - initializing RKE2 first server for compute_kube_control_plane_first_x86_64" + mv /tmp/ipaddress_pool.yaml {{ compute_k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + mv /tmp/l2advertisement.yaml {{ compute_k8s_client_mount_path }}/metallb/l2advertisement.yaml + + # Determine the VIP interface + NODE_IP="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + VIP_IFACE=$(ip -o addr show | awk -v ip="$NODE_IP" '$4 ~ ip {print $2}') + sed -i "s/value: vip_interface/value: ${VIP_IFACE}/" /tmp/kube-vip.yaml + + # Start RKE2 server + echo "Starting RKE2 server..." + systemctl enable rke2-server.service + systemctl start rke2-server.service + + # Wait for RKE2 to generate kubeconfig + echo "Waiting for RKE2 kubeconfig..." + while [ ! -f /etc/rancher/rke2/rke2.yaml ]; do + sleep 5 + done + + # Set up kubectl access + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + chmod 600 $HOME/.kube/config + + # Place kube-vip as a kubelet static pod now that rke2.yaml exists. + cp /tmp/kube-vip.yaml /var/lib/rancher/rke2/agent/pod-manifests/kube-vip.yaml + + echo "Waiting for one Ready control plane node ..." + while true; do + control_plane_ready=$(kubectl get nodes --no-headers 2>/dev/null | awk '$2=="Ready"' | wc -l) + if [ "$control_plane_ready" -ge 1 ]; then + echo "Found $control_plane_ready Ready node(s)!" + break + else + echo "No Ready node yet, waiting 5s ..." + sleep 5 + fi + done + + # Wait for all pods to be ready + echo "Waiting for all pods to be Ready (Running/Completed)..." + while true; do + not_ready=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '{ print $4 }' | grep -vE '^(Running|Completed)$' | wc -l) + if [ "${not_ready}" -eq 0 ]; then + echo "All pods are Running or Completed." + break + else + echo "$not_ready pods not yet ready, waiting 5s ..." + sleep 5 + fi + done + + # Save the server token to NFS for other nodes to join + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + while [ ! -f /var/lib/rancher/rke2/server/node-token ]; do + echo "Waiting for RKE2 node token..." + sleep 5 + done + cp /var/lib/rancher/rke2/server/node-token "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + chmod 644 "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + echo "RKE2 server token saved to NFS." + + # Update kubeconfig to use VIP + KUBE_VIP="{{ compute_kube_vip }}" + KUBE_PORT="6443" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" /etc/rancher/rke2/rke2.yaml + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + + # Also save kubeconfig to NFS for other control planes + cp /etc/rancher/rke2/rke2.yaml "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + sed -i "s|server: https://127.0.0.1:${KUBE_PORT}|server: https://${KUBE_VIP}:${KUBE_PORT}|" "${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-kubeconfig.yaml" + + echo "Waiting for at least one READY Kubernetes worker node ..." + while true; do + if kubectl get nodes --no-headers | grep -Ev 'control-plane|master' | grep ' Ready '; then + echo "Worker node(s) present and Ready." + break + else + echo "No Ready worker node detected yet. Retrying in 10 seconds..." + sleep 10 + fi + done + + echo "Installing plugins" + echo "Installing nfs-client-provisioner" + /usr/local/bin/helm install nfs-client {{ compute_k8s_client_mount_path }}/nfs-client-provisioner/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz \ + --namespace default --create-namespace \ + --set nfs.server={{ compute_k8s_nfs_server_path.split(':')[0] }} \ + --set nfs.path={{ compute_k8s_nfs_server_path.split(':')[1] }} \ + --set storageClass.defaultClass=true \ + --set storageClass.reclaimPolicy=Retain + echo "Waiting for nfs-subdir-external-provisioner pods to appear..." + sleep 15 + + if kubectl get pods -n default -l app=nfs-subdir-external-provisioner | grep -q nfs-subdir; then + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + else + echo "Pods not yet created, retrying after 10 seconds..." + sleep 10 + kubectl wait --for=condition=Ready pod -l app=nfs-subdir-external-provisioner -n default --timeout=300s || true + fi + + echo "Installing MetalLB" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/{{ compute_metallb_package }}.yml + echo "Waiting for MetalLB pods to be ready..." + kubectl wait --namespace metallb-system --for=condition=Ready pods --all --timeout=300s + echo "Waiting for MetalLB webhook to be ready..." + until kubectl get endpoints metallb-webhook-service -n metallb-system \ + -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -qE '[0-9]'; do + echo "Webhook endpoints not ready yet. Retrying in 5s..." + sleep 5 + done + echo "MetalLB webhook is ready." + echo "Deploy ipaddress pool" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/ipaddress_pool.yaml + echo "Deploy Layer2 Configuration" + kubectl create -f {{ compute_k8s_client_mount_path }}/metallb/l2advertisement.yaml + + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + + systemctl restart nfs-client.target + systemctl restart rpcbind + + echo "Display nodes and pods status after deploying plugins" + echo "Listing all Kubernetes nodes:" + kubectl get nodes -o wide + + echo "Listing all Kubernetes pods in all namespaces:" + kubectl get pods --all-namespaces -o wide + + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP INIT + echo "compute_kube_control_plane_first_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 new file mode 100644 index 0000000000..39053e6815 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_control_plane_x86_64.yaml.j2 @@ -0,0 +1,414 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + - path: /tmp/kube-vip.yaml + owner: root:root + permissions: '0644' + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + uid: kube-vip-pod + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_nodename + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: vip_interface + value: vip_interface + - name: vip_cidr + value: "{{ admin_netmask_bits }}" + - name: dns_mode + value: first + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: vip_address + value: {{ compute_kube_vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.8.9 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + volumes: + - hostPath: + path: /etc/rancher/rke2/rke2.yaml + name: kubeconfig + status: {} + + - path: /usr/local/bin/install-helm.sh + permissions: '0755' + content: | + #!/bin/bash + set -e + ARCH="amd64" + cp {{ compute_k8s_client_mount_path }}/helm/linux-${ARCH}/helm /usr/local/bin/helm + chmod +x /usr/local/bin/helm + /usr/local/bin/helm completion bash > /etc/bash_completion.d/helm.sh + chmod 0755 /etc/bash_completion.d/helm.sh + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open essential ports for RKE2 server + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=6443/tcp + - firewall-cmd --permanent --add-port=9345/tcp + - firewall-cmd --permanent --add-port=2379-2380/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=10257/tcp + - firewall-cmd --permanent --add-port=10259/tcp + + # CNI-related ports + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/pod-manifests /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - echo "Installing helm" + - /usr/local/bin/install-helm.sh + + - | + set -e + echo "Installing Necessary Python pip packages" + python3 -m ensurepip + + PACKAGES=({% for pkg in k8s_pip_packages %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %}) + + for pkg in "${PACKAGES[@]}"; do + echo "Installing $pkg from offline repo..." + pip3 install "$pkg" \ + --find-links="{{ offline_pip_module_path }}/${pkg}/" \ + --trusted-host "{{ pulp_server_ip }}" \ + --no-index + done + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + export PATH="$PATH:/var/lib/rancher/rke2/bin" + if [ ! -f "$MARKER" ]; then + # Join RKE2 cluster as additional server + echo "Initial boot - joining compute RKE2 cluster as additional server for compute_kube_control_plane_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ compute_kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the compute RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "Compute RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable." + + # Write RKE2 config for joining as server + cat > /etc/rancher/rke2/config.yaml </dev/null 2>&1; do + sleep 10 + done + kubectl get nodes -o wide + + systemctl restart nfs-client.target + systemctl restart rpcbind + touch "$MARKER" + echo "Cloud-Init has completed successfully." + else + # SUBSEQUENT BOOT - SKIP JOIN + echo "compute_kube_control_plane_x86_64 (RKE2) is already part of cluster." + echo "Cluster already initialized. Performing node reboot procedures." + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + export PATH="$PATH:/var/lib/rancher/rke2/bin" + ln -sf /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + mkdir -p $HOME/.kube /root/.kube + cp -f /etc/rancher/rke2/rke2.yaml $HOME/.kube/config + yes | cp -i /etc/rancher/rke2/rke2.yaml /root/.kube/config + systemctl enable rke2-server.service + systemctl start rke2-server.service + kubectl get nodes -o wide || echo "Cluster not yet fully up" + kubectl get pods --all-namespaces -o wide || echo "Pods may not be ready yet" + echo "Cloud-Init finished successfully after the reboot." + fi diff --git a/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 new file mode 100644 index 0000000000..b1cbe20075 --- /dev/null +++ b/provision/roles/configure_ochami/templates/cloud_init/rke2/ci-group-compute_kube_node_x86_64.yaml.j2 @@ -0,0 +1,260 @@ +- name: {{ functional_group_name }} + description: "{{ functional_group_name }}" + file: + encoding: plain + content: | + ## template: jinja + #cloud-config + + merge_how: + - name: list + settings: [append] + - name: dict + settings: [no_replace, recurse_list] + + users: + - name: root + ssh_authorized_keys: "{{ read_ssh_key.stdout }}" + lock_passwd: false + hashed_passwd: "{{ hashed_password_output.stdout }}" + disable_root: false + + write_files: + - path: /usr/local/bin/doca-install.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/doca-install.sh.j2') | indent(12) }} + + - path: /usr/local/bin/configure-ib-network.sh + owner: root:root + permissions: '{{ file_mode_755 }}' + content: | + {{ lookup('template', 'templates/doca-ofed/configure-ib-network.sh.j2') | indent(12) }} + + - path: /usr/local/bin/set-ssh.sh + permissions: '0755' + content: | + #!/bin/bash + timedatectl set-timezone {{ hostvars['oim']['oim_timezone'] }} + sed -i 's/^#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config + sed -i 's/^#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config + sed -i 's/^PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf + systemctl restart sshd + default_count=$(ip route | grep -c "^default") + if [ "$default_count" -le 1 ]; then + echo "Only one or no default route found. No action needed." + else + private_nic=$(ip route | grep "^default via {{ hostvars['localhost']['admin_nic_ip'] }}" | awk '{print $5}') + # Get all default routes + ip route | grep '^default' | while read -r line; do + nmcli con del "Wired Connection" + # Extract NIC name + nic=$(echo "$line" | awk '{print $5}') + + # Add the default route to the connection + if [ -n "$nic" ]; then + echo "Adding nmcli device $nic" + nmcli con add type ethernet ifname "$nic" con-name "$nic" ipv4.method auto + if [ "$nic" = "$private_nic" ]; then + nmcli con modify "$nic" ipv4.never-default yes + nmcli con delete "cloud-init $nic" + fi + nmcli con up "$nic" + else + echo "No connection found for device $nic" + fi + done + fi + + - path: /root/.ssh/config + permissions: '0600' + content: | + Host * + IdentityFile {{ compute_k8s_client_mount_path }}/ssh/oim_rsa + IdentitiesOnly yes + + - path: /etc/modules-load.d/k8s.conf + content: | + br_netfilter + overlay + nf_conntrack + vxlan + permissions: '0644' + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-iptables=1 + net.bridge.bridge-nf-call-ip6tables=1 + net.ipv4.ip_forward=1 + vm.overcommit_memory=1 + kernel.panic=10 + permissions: '0644' + - path: /etc/fstab + content: | + {{ compute_k8s_nfs_server_path }} {{ compute_k8s_client_mount_path }} nfs noatime,nolock 0 0 + {{ compute_k8s_nfs_server_path }}/{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}/pod-logs /var/log/pods nfs noatime,nolock 0 0 + permissions: '0644' + + - path: /etc/rancher/rke2/registries.yaml + owner: root:root + permissions: '0644' + content: | + mirrors: + docker.io: + endpoint: + - "https://{{ pulp_mirror }}" + ghcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + quay.io: + endpoint: + - "https://{{ pulp_mirror }}" + registry.k8s.io: + endpoint: + - "https://{{ pulp_mirror }}" + nvcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + public.ecr.aws: + endpoint: + - "https://{{ pulp_mirror }}" + gcr.io: + endpoint: + - "https://{{ pulp_mirror }}" + configs: + "{{ pulp_mirror }}": + tls: + insecure_skip_verify: true + + runcmd: + - /usr/local/bin/set-ssh.sh + - "systemctl enable chronyd" + - "systemctl restart chronyd" + - "chronyc sources" + - "chronyc -a makestep" + - sudo swapoff -a + - sudo sed -i '/ swap / s/^/#/' /etc/fstab + - sudo setenforce 0 || true + - sudo sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config + + # Enable and start firewalld + - systemctl enable firewalld + - systemctl start firewalld + + # Open required ports for RKE2 agent + - firewall-cmd --permanent --add-port=22/tcp + - firewall-cmd --permanent --add-port=10250/tcp + - firewall-cmd --permanent --add-port=30000-32767/tcp + - firewall-cmd --permanent --add-port=179/tcp + - firewall-cmd --permanent --add-port=4789/udp + - firewall-cmd --permanent --add-port=5473/tcp + - firewall-cmd --permanent --add-port=8472/udp + - firewall-cmd --permanent --add-port=51820/udp + - firewall-cmd --permanent --add-port=51821/udp + - firewall-cmd --permanent --add-port=9100/tcp + - firewall-cmd --permanent --add-port=7472/tcp + - firewall-cmd --permanent --add-port=7472/udp + - firewall-cmd --permanent --add-port=7946/tcp + - firewall-cmd --permanent --add-port=7946/udp + - firewall-cmd --permanent --add-port=9090/tcp + - firewall-cmd --permanent --add-port=8080/tcp + - firewall-cmd --permanent --add-service=http + - firewall-cmd --permanent --add-service=https + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_service_addresses }} + - firewall-cmd --permanent --zone=trusted --add-source={{ compute_k8s_pod_network_cidr }} + - firewall-cmd --set-default-zone=trusted + - firewall-cmd --reload + + - sudo modprobe br_netfilter || true + - sudo modprobe overlay || true + - sudo modprobe nf_conntrack || true + - sudo modprobe vxlan || true + - sysctl --system + - mkdir -p {{ compute_k8s_client_mount_path }} /var/log/pods /etc/rancher/rke2 /var/lib/rancher/rke2/agent/images + - | + tmpfile=$(mktemp) + + # Extract the first 'search' line only (ignore duplicates) + search_line=$(grep '^search' /etc/resolv.conf | head -n1) + [ -n "$search_line" ] && echo "$search_line" > "$tmpfile" + + # Add your new nameserver entries + {% for ns in dns %} + echo "nameserver {{ ns }}" >> "$tmpfile" + {% endfor %} + + # Add remaining lines except search and empty lines + grep -v '^search' /etc/resolv.conf | grep -v '^$' >> "$tmpfile" + + # Remove duplicate lines + awk '!seen[$0]++' "$tmpfile" > /etc/resolv.conf + - | + if command -v chattr >/dev/null 2>&1; then + chattr +i /etc/resolv.conf || true + fi + - systemctl restart rpcbind + - mount -a + # Stage RKE2 airgap system images (core + CNI); RKE2 auto-imports them on start + - cp {{ compute_k8s_client_mount_path }}/rke2/images/*.tar.zst /var/lib/rancher/rke2/agent/images/ + - cp {{ compute_k8s_client_mount_path }}/pulp_webserver.crt /etc/pki/ca-trust/source/anchors + - update-ca-trust extract + - sed -i 's/^gpgcheck=1/gpgcheck=0/' /etc/dnf/dnf.conf + - bash /usr/local/bin/doca-install.sh && bash /usr/local/bin/configure-ib-network.sh + - | + set -e + MARKER="/etc/rancher/rke2/.compute_cluster_initialized" + if [ ! -f "$MARKER" ]; then + # Join compute RKE2 cluster as agent (worker node) + echo "Initial boot - joining compute RKE2 cluster as agent for compute_kube_node_x86_64" + K8S_CLIENT_MOUNT_PATH="{{ compute_k8s_client_mount_path }}" + NODE_NAME="{% raw %}{{ ds.meta_data.instance_data.local_ipv4 }}{% endraw %}" + KUBE_VIP="{{ compute_kube_vip }}" + TOKEN_FILE="${K8S_CLIENT_MOUNT_PATH}/rke2/rke2-server-token" + + echo "----------------------------------------------------------------------" + echo "Waiting for the compute RKE2 first server to be initialized." + echo "This node will automatically join the cluster once the token is available." + echo "Looking for server token at: $TOKEN_FILE" + echo "----------------------------------------------------------------------" + while [ ! -f "$TOKEN_FILE" ]; do + echo "Compute RKE2 first server is not ready yet. Waiting for $TOKEN_FILE. Retrying in 10 seconds..." + sleep 10 + done + echo "Token file detected: $TOKEN_FILE" + RKE2_TOKEN=$(cat "$TOKEN_FILE") + + echo "Checking if kube-vip (${KUBE_VIP}) is reachable..." + while ! ping -c 1 -W 2 "$KUBE_VIP" >/dev/null 2>&1; do + echo "kube-vip (${KUBE_VIP}) not reachable. Retrying in 10 seconds..." + sleep 10 + done + echo "kube-vip (${KUBE_VIP}) is reachable. Joining this compute_kube_node_x86_64 to the cluster now." + + # Write RKE2 agent config + cat > /etc/rancher/rke2/config.yaml <- + {{ cp_first_cluster | selectattr('type', 'equalto', 'manifest') + | selectattr('package', 'search', 'metallb-native') + | map(attribute='package') | join }} + compute_helm_package: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'helm') + | map(attribute='package') | join }} + compute_nfs_subdir_external_provisioner_pkg: >- + {{ cp_first_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'nfs-subdir-external-provisioner') + | map(attribute='package') | join }} + +- name: Extract compute RKE2 airgap image tarball package names # noqa: yaml[line-length] + vars: + rke2_cluster: "{{ compute_k8s_packages_json['compute_rke2']['cluster'] }}" + ansible.builtin.set_fact: + compute_rke2_images_core_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-core') + | map(attribute='package') | join }} + compute_rke2_images_cni_pkg: >- + {{ rke2_cluster | selectattr('type', 'equalto', 'tarball') + | selectattr('package', 'search', 'rke2-images-' ~ compute_k8s_cni) + | map(attribute='package') | join }} + +- name: Copy pulp webserver certificate to target host + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ anchors_path }}" + mode: "{{ file_mode }}" + become: true + +- name: Update CA trust on target host + ansible.builtin.command: update-ca-trust + register: update_ca + changed_when: false + +# metallb +- name: Download metallb-native manifest for compute cluster + ansible.builtin.get_url: + url: "{{ offline_manifest_path }}/{{ compute_metallb_package }}/{{ compute_metallb_package }}.yml" + dest: "{{ compute_k8s_client_mount_path }}/metallb/{{ compute_metallb_package }}.yml" + mode: "{{ file_mode }}" + +# helm +- name: Download helm tarball for compute cluster + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ compute_helm_package }}/{{ compute_helm_package }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/helm/{{ compute_helm_package }}.tar.gz" + mode: "{{ file_mode }}" + +- name: Untar helm tarball repo for compute cluster + ansible.builtin.unarchive: + src: "{{ compute_k8s_client_mount_path }}/helm/{{ compute_helm_package }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/helm/" + remote_src: true + +# nfs client provisioner +- name: Download nfs-client-provisioner tarball for compute cluster + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ compute_nfs_subdir_external_provisioner_pkg }}/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/nfs-client-provisioner/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" + mode: "{{ file_mode }}" + +# RKE2 airgap system images (core + selected CNI) +- name: Download RKE2 airgap image tarballs to compute NFS share + ansible.builtin.get_url: + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" + dest: "{{ compute_k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + mode: "{{ file_mode }}" + loop: + - "{{ compute_rke2_images_core_pkg }}" + - "{{ compute_rke2_images_cni_pkg }}" + +- name: Copy pulp webserver certificate to compute client_share_path + ansible.builtin.copy: + src: "{{ pulp_webserver_cert_path }}" + dest: "{{ compute_k8s_client_mount_path }}" + mode: "{{ file_mode }}" + become: true diff --git a/provision/roles/k8s_config/tasks/main.yml b/provision/roles/k8s_config/tasks/main.yml index 5d7cb32686..b342df0007 100644 --- a/provision/roles/k8s_config/tasks/main.yml +++ b/provision/roles/k8s_config/tasks/main.yml @@ -29,3 +29,8 @@ when: - hostvars['localhost']['service_k8s_support'] - k8s_distro | default('kubeadm') == 'rke2' + +- name: Creating the configuration files required for compute RKE2 cluster in nfs share + ansible.builtin.include_tasks: create_compute_rke2_config_nfs.yml + when: + - hostvars['localhost']['compute_rke2_support'] | default(false) | bool diff --git a/provision/roles/k8s_config/vars/main.yml b/provision/roles/k8s_config/vars/main.yml index 3551f4da96..875ea84430 100644 --- a/provision/roles/k8s_config/vars/main.yml +++ b/provision/roles/k8s_config/vars/main.yml @@ -18,6 +18,7 @@ input_project_dir: "{{ hostvars['localhost']['input_project_dir'] }}" # Versioned JSON file: service_k8s_v.json (e.g., service_k8s_v1.35.1.json) k8s_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_k8s_v{{ hostvars['localhost']['service_k8s_version'] }}.json" # noqa: yaml[line-length] rke2_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/service_rke2.json" # noqa: yaml[line-length] +compute_rke2_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/compute_rke2.json" # noqa: yaml[line-length] calico_manifest_yaml_url: "{{ offline_manifest_path }}/{{ calico_package }}/{{ calico_package }}.yml" csi_powerscale_packages_file: "{{ input_project_dir }}/config/x86_64/{{ software_config.cluster_os_type }}/{{ software_config.cluster_os_version }}/csi_driver_powerscale.json" # noqa: yaml[line-length] metallb_manifest_yaml_url: "{{ offline_manifest_path }}/{{ metallb_package }}/{{ metallb_package }}.yml" diff --git a/provision/roles/provision_validations/tasks/include_software_config.yml b/provision/roles/provision_validations/tasks/include_software_config.yml index 652d6620c0..dc53ea2bf4 100644 --- a/provision/roles/provision_validations/tasks/include_software_config.yml +++ b/provision/roles/provision_validations/tasks/include_software_config.yml @@ -54,6 +54,7 @@ ansible.builtin.set_fact: service_k8s_support: false service_rke2_support: false + compute_rke2_support: false - name: Check if service k8s support is true ansible.builtin.set_fact: @@ -63,6 +64,10 @@ ansible.builtin.set_fact: service_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'service_rke2') | list | length > 0 }}" +- name: Check if compute rke2 support is true + ansible.builtin.set_fact: + compute_rke2_support: "{{ software_config.softwares | selectattr('name', 'equalto', 'compute_rke2') | list | length > 0 }}" + - name: Set service_k8s_support true when rke2 is selected ansible.builtin.set_fact: service_k8s_support: true From caff2d6f97b778aca546f0e2e1c0ca4b57cffff8 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Mon, 6 Jul 2026 15:01:28 -0500 Subject: [PATCH 05/13] fix: use standard repo names in compute_rke2.json Change x86_64_baseos to baseos and x86_64_appstream to appstream to match the naming convention used by service_k8s_v1.35.1.json and the RHEL subscription repo configuration. Signed-off-by: John Lockman --- input/config/x86_64/rhel/10.0/compute_rke2.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/input/config/x86_64/rhel/10.0/compute_rke2.json b/input/config/x86_64/rhel/10.0/compute_rke2.json index 263b728b5b..18164b3f9c 100644 --- a/input/config/x86_64/rhel/10.0/compute_rke2.json +++ b/input/config/x86_64/rhel/10.0/compute_rke2.json @@ -2,10 +2,10 @@ "compute_rke2": { "cluster": [ { "package": "docker.io/library/busybox", "type": "image", "tag": "1.36" }, - { "package": "firewalld", "type": "rpm", "repo_name": "x86_64_baseos" }, - { "package": "python3-firewall", "type": "rpm", "repo_name": "x86_64_baseos" }, - { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, - { "package": "vim", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "firewalld", "type": "rpm", "repo_name": "baseos" }, + { "package": "python3-firewall", "type": "rpm", "repo_name": "baseos" }, + { "package": "git", "type": "rpm", "repo_name": "appstream"}, + { "package": "vim", "type": "rpm", "repo_name": "appstream"}, { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, @@ -24,8 +24,8 @@ { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, { "package": "prettytable==3.14.0", "type": "pip_module" }, - { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, - { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "python3.12", "type": "rpm", "repo_name": "appstream" }, + { "package": "git", "type": "rpm", "repo_name": "appstream"}, { "package": "kubernetes==33.1.0", "type": "pip_module" } ] }, @@ -41,8 +41,8 @@ { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, { "package": "prettytable==3.14.0", "type": "pip_module" }, - { "package": "python3.12", "type": "rpm", "repo_name": "x86_64_appstream" }, - { "package": "git", "type": "rpm", "repo_name": "x86_64_appstream"}, + { "package": "python3.12", "type": "rpm", "repo_name": "appstream" }, + { "package": "git", "type": "rpm", "repo_name": "appstream"}, { "package": "kubernetes==33.1.0", "type": "pip_module" } ] }, From 14bba2e11aeaf4496331388f612e8cbae95dbf78 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Tue, 7 Jul 2026 11:13:42 -0500 Subject: [PATCH 06/13] fix: skip URL reachability check for rancher.io repos rpm.rancher.io returns HTTP 403 to Python urllib default User-Agent, causing the repo URL validation to fail even though the repo is accessible. Add 'rancher' to the skip list alongside k8s, cri-o, and other CDNs that block automated clients. Signed-off-by: John Lockman --- common/library/module_utils/local_repo/software_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/library/module_utils/local_repo/software_utils.py b/common/library/module_utils/local_repo/software_utils.py index af3c1ffab9..97000faac2 100644 --- a/common/library/module_utils/local_repo/software_utils.py +++ b/common/library/module_utils/local_repo/software_utils.py @@ -522,7 +522,7 @@ def parse_repo_urls(repo_config, local_repo_config_path, seen_urls.add(rendered_url) # # Skip reachability check for URLs containing k8s, cri-o, oneapi, snoopy, nvidia - if not any(skip_str in rendered_url for skip_str in ["k8s", "cri-o", "oneapi", "snoopy", "nvidia"]): + if not any(skip_str in rendered_url for skip_str in ["k8s", "cri-o", "oneapi", "snoopy", "nvidia", "rancher"]): if not is_remote_url_reachable(rendered_url): logger.error(f"OMNIA repo URL unreachable: {rendered_url}") return rendered_url, False From 5f73113cc7a01ecad2f7a896a154ad43c289aca3 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Thu, 9 Jul 2026 16:20:14 -0500 Subject: [PATCH 07/13] fix: add sslverify=0 to generated pulp.repo entries Pulp uses self-signed SSL certificates. The generated repo file in /etc/yum.repos.d/pulp.repo must include sslverify=0 for each section, otherwise dnf makecache fails with SSL verification errors when Ansible regenerates the DNF cache. Signed-off-by: John Lockman --- common/library/modules/process_rpm_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common/library/modules/process_rpm_config.py b/common/library/modules/process_rpm_config.py index 34d650d4a3..fb90123a20 100644 --- a/common/library/modules/process_rpm_config.py +++ b/common/library/modules/process_rpm_config.py @@ -803,6 +803,7 @@ def create_yum_repo_file(distributions, log): baseurl={base_url} enabled=1 gpgcheck=0 +sslverify=0 """ repo_content += repo_entry.strip() + "\n\n" From aec126fe14c5800bc09985d0e9ee50d2afb44a65 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Thu, 9 Jul 2026 17:03:57 -0500 Subject: [PATCH 08/13] fix: SSL verify and package names for compute_rke2 - download_common.py: use verify=False for Pulp self-signed cert downloads instead of verify=PULP_SSL_CA_CERT which fails with self-signed certificates - compute_rke2.json: fix package names for RHEL 10 - vim -> vim-enhanced (correct RPM name) - python3.12 -> python3 (RHEL 10 default Python package) Signed-off-by: John Lockman --- common/library/module_utils/local_repo/download_common.py | 2 +- input/config/x86_64/rhel/10.0/compute_rke2.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/library/module_utils/local_repo/download_common.py b/common/library/module_utils/local_repo/download_common.py index 40dbff1c86..d0b9f11b5b 100644 --- a/common/library/module_utils/local_repo/download_common.py +++ b/common/library/module_utils/local_repo/download_common.py @@ -181,7 +181,7 @@ def sanitize_path(path: str) -> str: full_url, stream=True, headers=headers, - verify=PULP_SSL_CA_CERT, + verify=False, timeout=(30, 600) ) as r: diff --git a/input/config/x86_64/rhel/10.0/compute_rke2.json b/input/config/x86_64/rhel/10.0/compute_rke2.json index 18164b3f9c..3ecd262296 100644 --- a/input/config/x86_64/rhel/10.0/compute_rke2.json +++ b/input/config/x86_64/rhel/10.0/compute_rke2.json @@ -5,7 +5,7 @@ { "package": "firewalld", "type": "rpm", "repo_name": "baseos" }, { "package": "python3-firewall", "type": "rpm", "repo_name": "baseos" }, { "package": "git", "type": "rpm", "repo_name": "appstream"}, - { "package": "vim", "type": "rpm", "repo_name": "appstream"}, + { "package": "vim-enhanced", "type": "rpm", "repo_name": "appstream"}, { "package": "rke2-server", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-agent", "type": "rpm", "repo_name": "rke2" }, { "package": "rke2-common", "type": "rpm", "repo_name": "rke2" }, @@ -24,7 +24,7 @@ { "package": "ghcr.io/kube-vip/kube-vip", "tag": "v0.8.9", "type": "image" }, { "package": "quay.io/metallb/speaker", "tag": "v0.15.2", "type": "image" }, { "package": "prettytable==3.14.0", "type": "pip_module" }, - { "package": "python3.12", "type": "rpm", "repo_name": "appstream" }, + { "package": "python3", "type": "rpm", "repo_name": "appstream" }, { "package": "git", "type": "rpm", "repo_name": "appstream"}, { "package": "kubernetes==33.1.0", "type": "pip_module" } ] @@ -41,7 +41,7 @@ { "package": "helm-v3.19.0-amd64", "type": "tarball", "url": "https://get.helm.sh/helm-v3.19.0-linux-amd64.tar.gz" }, { "package": "nfs-subdir-external-provisioner-4.0.18", "type": "tarball", "url": "https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/releases/download/nfs-subdir-external-provisioner-4.0.18/nfs-subdir-external-provisioner-4.0.18.tgz" }, { "package": "prettytable==3.14.0", "type": "pip_module" }, - { "package": "python3.12", "type": "rpm", "repo_name": "appstream" }, + { "package": "python3", "type": "rpm", "repo_name": "appstream" }, { "package": "git", "type": "rpm", "repo_name": "appstream"}, { "package": "kubernetes==33.1.0", "type": "pip_module" } ] From 3b0ec60efd12c5b62214c7503270070f289e643c Mon Sep 17 00:00:00 2001 From: John Lockman Date: Thu, 9 Jul 2026 17:40:32 -0500 Subject: [PATCH 09/13] fix: include compute_rke2 in HA config name extraction The HA cluster name extraction loop only processed service_k8s_cluster_ha, causing the cluster_name lookup for compute_rke2_k8s_cluster_ha to fail (comparing string against list of dicts instead of list of names). Signed-off-by: John Lockman --- .../input_validation/validation_flows/common_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/library/module_utils/input_validation/validation_flows/common_validation.py b/common/library/module_utils/input_validation/validation_flows/common_validation.py index 52c6ce9e0c..54f2f13667 100644 --- a/common/library/module_utils/input_validation/validation_flows/common_validation.py +++ b/common/library/module_utils/input_validation/validation_flows/common_validation.py @@ -1687,7 +1687,7 @@ def validate_omnia_config( input_file_path, file_names["high_availability_config"]) with open(ha_config_path, "r", encoding="utf-8") as f: ha_config = yaml.safe_load(f) - for k in ["service_k8s_cluster_ha"]: + for k in ["service_k8s_cluster_ha", "compute_rke2_k8s_cluster_ha"]: ha_config[k] = [xha["cluster_name"] for xha in ha_config.get(k, [])] validate_k8s(data, admin_networks, sw_list, ha_config, tag_names, errors, st_config, module, input_file_path, logger) From 91e2596fb8da11f430c01c2108e9f872b155e3a4 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Mon, 13 Jul 2026 08:26:11 -0500 Subject: [PATCH 10/13] fix: disable SSL cert verification for Pulp local mirror downloads Pulp generates self-signed SSL certificates for its HTTPS content server. Multiple Ansible get_url tasks and image-build config templates download manifests, tarballs, and packages from Pulp without setting validate_certs: false, causing SSL verification failures. Affected areas: - provision/k8s_config: 15 get_url tasks across create_k8s_config_nfs, create_rke2_config_nfs, create_compute_rke2_config_nfs, and get_powerscale_dependencies - build_image_x86_64: 2 config templates (rhel-base-config.yaml.j2, rhel-compute-config.yaml.j2) that generate repo configs used by dnf inside the image-build container This is safe because all affected downloads are from the local Pulp mirror (192.168.x.x:2225), not external sources. Signed-off-by: John Lockman --- .../image_creation/templates/images/rhel-base-config.yaml.j2 | 1 + .../templates/images/rhel-compute-config.yaml.j2 | 1 + .../roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml | 4 ++++ provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml | 4 ++++ provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml | 4 ++++ .../roles/k8s_config/tasks/get_powerscale_dependencies.yml | 3 +++ 6 files changed, 17 insertions(+) diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 index 1b75508858..0734b2e34b 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-base-config.yaml.j2 @@ -13,6 +13,7 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' + sslverify: false {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 index a10790edf7..1bfaa885e1 100644 --- a/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 +++ b/build_image_x86_64/roles/image_creation/templates/images/rhel-compute-config.yaml.j2 @@ -18,6 +18,7 @@ repos: {% if repo.base_url | length > 1 %} - alias: '{{ repo.name }}' url: '{{ repo.base_url }}' + sslverify: false {% endif %} {% if repo.gpg | length > 1 %} gpg: '{{ repo.gpg }}' diff --git a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml index 9a9dc39b21..c1c253d4f5 100644 --- a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml @@ -158,6 +158,7 @@ url: "{{ offline_manifest_path }}/{{ compute_metallb_package }}/{{ compute_metallb_package }}.yml" dest: "{{ compute_k8s_client_mount_path }}/metallb/{{ compute_metallb_package }}.yml" mode: "{{ file_mode }}" + validate_certs: false # helm - name: Download helm tarball for compute cluster @@ -165,6 +166,7 @@ url: "{{ offline_tarball_path }}/{{ compute_helm_package }}/{{ compute_helm_package }}.tar.gz" dest: "{{ compute_k8s_client_mount_path }}/helm/{{ compute_helm_package }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false - name: Untar helm tarball repo for compute cluster ansible.builtin.unarchive: @@ -178,6 +180,7 @@ url: "{{ offline_tarball_path }}/{{ compute_nfs_subdir_external_provisioner_pkg }}/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" dest: "{{ compute_k8s_client_mount_path }}/nfs-client-provisioner/{{ compute_nfs_subdir_external_provisioner_pkg }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false # RKE2 airgap system images (core + selected CNI) - name: Download RKE2 airgap image tarballs to compute NFS share @@ -185,6 +188,7 @@ url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" dest: "{{ compute_k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" mode: "{{ file_mode }}" + validate_certs: false loop: - "{{ compute_rke2_images_core_pkg }}" - "{{ compute_rke2_images_cni_pkg }}" diff --git a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml index 87f29eb7bf..cd92584611 100644 --- a/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_k8s_config_nfs.yml @@ -252,6 +252,7 @@ url: "{{ calico_manifest_yaml_url }}" dest: "{{ k8s_client_mount_path }}/calico/{{ calico_package }}.yml" mode: "{{ file_mode }}" + validate_certs: false # metallb - name: Download metallb-native manifest @@ -259,6 +260,7 @@ url: "{{ metallb_manifest_yaml_url }}" dest: "{{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml" mode: "{{ file_mode }}" + validate_certs: false # multus # - name: Download multus manifest @@ -273,6 +275,7 @@ url: "{{ helm_tarball_url }}" dest: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false - name: Extract helm version from package name ansible.builtin.set_fact: @@ -299,6 +302,7 @@ url: "{{ nfs_client_provisioner_tarball_url }}" dest: "{{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false # whereabouts # - name: Get whereabouts plugin git folder diff --git a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml index 490a99fa2d..3942293c46 100644 --- a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml @@ -247,6 +247,7 @@ url: "{{ metallb_manifest_yaml_url }}" dest: "{{ k8s_client_mount_path }}/metallb/{{ metallb_package }}.yml" mode: "{{ file_mode }}" + validate_certs: false # helm - name: Download helm tarball @@ -254,6 +255,7 @@ url: "{{ helm_tarball_url }}" dest: "{{ k8s_client_mount_path }}/helm/{{ helm_package }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false - name: Untar helm tarball repo ansible.builtin.unarchive: @@ -267,6 +269,7 @@ url: "{{ nfs_client_provisioner_tarball_url }}" dest: "{{ k8s_client_mount_path }}/nfs-client-provisioner/{{ nfs_subdir_external_provisioner_pkg }}.tar.gz" mode: "{{ file_mode }}" + validate_certs: false # RKE2 airgap system images (core + selected CNI) # RKE2 auto-imports any *.tar.zst found in /var/lib/rancher/rke2/agent/images/ @@ -276,6 +279,7 @@ url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" dest: "{{ k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" mode: "{{ file_mode }}" + validate_certs: false loop: - "{{ rke2_images_core_pkg }}" - "{{ rke2_images_cni_pkg }}" diff --git a/provision/roles/k8s_config/tasks/get_powerscale_dependencies.yml b/provision/roles/k8s_config/tasks/get_powerscale_dependencies.yml index f17cdb4b06..b2aaa12ea9 100644 --- a/provision/roles/k8s_config/tasks/get_powerscale_dependencies.yml +++ b/provision/roles/k8s_config/tasks/get_powerscale_dependencies.yml @@ -129,6 +129,7 @@ url: "{{ offline_git_path }}/{{ csi_powerscale_dir }}/{{ csi_powerscale_dir }}.tar.gz" dest: "{{ k8s_client_mount_path }}/{{ csi_powerscale_dir }}.tar.gz" mode: "{{ permission_644 }}" + validate_certs: false - name: Extract csi-powerscale tar file ansible.builtin.unarchive: @@ -141,12 +142,14 @@ url: "{{ offline_git_path }}/{{ helm_charts_dir }}/{{ helm_charts_dir }}.tar.gz" dest: "{{ k8s_client_mount_path }}/{{ csi_powerscale_dir }}/{{ helm_charts_git }}" mode: "{{ permission_644 }}" + validate_certs: false - name: Get external-snapshotter git tar ansible.builtin.get_url: url: "{{ offline_git_path }}/{{ external_snapshotter_dir }}/{{ external_snapshotter_dir }}.tar.gz" dest: "{{ k8s_client_mount_path }}/{{ csi_powerscale_dir }}/{{ external_snapshotter_git }}" mode: "{{ permission_644 }}" + validate_certs: false - name: Transfer storage class template to nfs share (inside versioned directory) ansible.builtin.template: From 4bd50bc34fee85b63b845046d49e292e422fde9b Mon Sep 17 00:00:00 2001 From: John Lockman Date: Mon, 13 Jul 2026 09:08:08 -0500 Subject: [PATCH 11/13] fix: add compute_rke2_k8s_cluster_ha to HA config JSON schema The high_availability_config.json schema only allows service_k8s_cluster_ha as a property. Add compute_rke2_k8s_cluster_ha with the same structure (cluster_name, enable_k8s_ha, virtual_ip_address) so the compute RKE2 HA configuration passes schema validation. Signed-off-by: John Lockman --- .../schema/high_availability_config.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/common/library/module_utils/input_validation/schema/high_availability_config.json b/common/library/module_utils/input_validation/schema/high_availability_config.json index c868eb2c5e..747a8d4792 100644 --- a/common/library/module_utils/input_validation/schema/high_availability_config.json +++ b/common/library/module_utils/input_validation/schema/high_availability_config.json @@ -34,6 +34,39 @@ ], "additionalProperties": false } + }, + "compute_rke2_k8s_cluster_ha": { + "type": "array", + "description": "High Availability (HA) configuration for the compute RKE2 Kubernetes cluster.", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "cluster_name": { + "type": "string", + "minLength": 1, + "description": "Required. Name of the compute RKE2 cluster. Must match the cluster name in omnia_config.yml." + }, + "enable_k8s_ha": { + "type": "boolean", + "description": "Mandatory. Enable HA for the compute RKE2 cluster." + }, + "virtual_ip_address": { + "type": "string", + "description": "Mandatory. Virtual IP address for the compute RKE2 cluster API endpoint.", + "allOf": [ + { "pattern": "^[0-9.]+$" }, + { "format": "ipv4" } + ] + } + }, + "required": [ + "cluster_name", + "enable_k8s_ha", + "virtual_ip_address" + ], + "additionalProperties": false + } } }, "required": [ From c52234b36af73e9016d9e8f855483564cfa6a69a Mon Sep 17 00:00:00 2001 From: John Lockman Date: Mon, 13 Jul 2026 11:18:31 -0500 Subject: [PATCH 12/13] fix: use .tar.gz extension for RKE2 airgap tarballs from Pulp Pulp downloads and stores RKE2 airgap image tarballs as .tar.gz regardless of the original .tar.zst extension in the source URL. Update the NFS config download tasks to use .tar.gz when fetching from the local Pulp mirror. Signed-off-by: John Lockman --- .../roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml | 4 ++-- provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml index c1c253d4f5..42bc27aafd 100644 --- a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml @@ -185,8 +185,8 @@ # RKE2 airgap system images (core + selected CNI) - name: Download RKE2 airgap image tarballs to compute NFS share ansible.builtin.get_url: - url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" - dest: "{{ compute_k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.gz" + dest: "{{ compute_k8s_client_mount_path }}/rke2/images/{{ item }}.tar.gz" mode: "{{ file_mode }}" validate_certs: false loop: diff --git a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml index 3942293c46..bc5b594295 100644 --- a/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_rke2_config_nfs.yml @@ -276,8 +276,8 @@ # on the nodes; the cloud-init templates copy these from the NFS share. - name: Download RKE2 airgap image tarballs to NFS share ansible.builtin.get_url: - url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.zst" - dest: "{{ k8s_client_mount_path }}/rke2/images/{{ item }}.tar.zst" + url: "{{ offline_tarball_path }}/{{ item }}/{{ item }}.tar.gz" + dest: "{{ k8s_client_mount_path }}/rke2/images/{{ item }}.tar.gz" mode: "{{ file_mode }}" validate_certs: false loop: From 9ba8cd6e1e4f4d2a456f5ac2ea2af519825b94c5 Mon Sep 17 00:00:00 2001 From: John Lockman Date: Tue, 14 Jul 2026 10:50:44 -0500 Subject: [PATCH 13/13] fix: replace ignore_errors with failed_when for ansible-lint compliance ansible-lint ignore-errors rule requires failed_when instead of ignore_errors for explicit error condition specification. Signed-off-by: John Lockman --- .../roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml index 42bc27aafd..d742ed1445 100644 --- a/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml +++ b/provision/roles/k8s_config/tasks/create_compute_rke2_config_nfs.yml @@ -66,7 +66,7 @@ compute_enable_k8s_ha: "{{ compute_rke2_k8s_cluster_ha[0].enable_k8s_ha | default(false) }}" compute_kube_vip: "{{ compute_rke2_k8s_cluster_ha[0].virtual_ip_address | default('') }}" when: compute_rke2_k8s_cluster_ha is defined - ignore_errors: true + failed_when: false - name: Set default HA facts when compute_rke2_k8s_cluster_ha is not defined ansible.builtin.set_fact: