From 5a6f6f02312aa93fa334892d92afb3891b9a00ad Mon Sep 17 00:00:00 2001 From: Jude Zhu Date: Fri, 24 Jul 2026 14:56:49 -0700 Subject: [PATCH] fix(nodepool): add CAPI autoscaler capacity annotations for scale-from-zero HyperShift deploys cluster-autoscaler with --cloud-provider=clusterapi, which reads capacity.cluster-autoscaler.kubernetes.io/* annotation keys. The scale-from-zero workaround only wrote machine.openshift.io/* keys (Machine API provider namespace), which the CAPI provider ignores. Add correct CAPI annotation keys alongside existing legacy keys: - capacity.cluster-autoscaler.kubernetes.io/cpu - capacity.cluster-autoscaler.kubernetes.io/memory (with MiB suffix) - capacity.cluster-autoscaler.kubernetes.io/gpu-count Legacy machine.openshift.io/* keys are preserved for backward compatibility and will be deprecated in a follow-up. Bug: https://issues.redhat.com/browse/OCPBUGS-99759 --- .../controllers/nodepool/scale_from_zero.go | 30 ++++++++++++++----- .../nodepool/scale_from_zero_test.go | 27 +++++++++++++++-- test/e2e/autoscaling_test.go | 24 ++++++++++----- 3 files changed, 63 insertions(+), 18 deletions(-) diff --git a/hypershift-operator/controllers/nodepool/scale_from_zero.go b/hypershift-operator/controllers/nodepool/scale_from_zero.go index 10658b2c5be2..cc8d11196866 100644 --- a/hypershift-operator/controllers/nodepool/scale_from_zero.go +++ b/hypershift-operator/controllers/nodepool/scale_from_zero.go @@ -32,12 +32,20 @@ import ( ) const ( - // Annotation keys for scale-from-zero workaround + // Legacy annotation keys (Machine API provider, --cloud-provider=openshift) cpuKey = "machine.openshift.io/vCPU" memoryKey = "machine.openshift.io/memoryMb" gpuKey = "machine.openshift.io/GPU" - labelsKey = "capacity.cluster-autoscaler.kubernetes.io/labels" - taintsKey = "capacity.cluster-autoscaler.kubernetes.io/taints" + + // CAPI autoscaler annotation keys (--cloud-provider=clusterapi) + capiCPUKey = "capacity.cluster-autoscaler.kubernetes.io/cpu" + capiMemoryKey = "capacity.cluster-autoscaler.kubernetes.io/memory" + capiGPUCountKey = "capacity.cluster-autoscaler.kubernetes.io/gpu-count" + capiGPUTypeKey = "capacity.cluster-autoscaler.kubernetes.io/gpu-type" + labelsKey = "capacity.cluster-autoscaler.kubernetes.io/labels" + taintsKey = "capacity.cluster-autoscaler.kubernetes.io/taints" + + defaultGPUResourceName = "nvidia.com/gpu" archLabelKey = "kubernetes.io/arch" ) @@ -91,7 +99,7 @@ func setScaleFromZeroAnnotationsOnObject(ctx context.Context, provider instancet // 2. Check if Status.Capacity is already provided (prefer native support) if len(statusCapacity) > 0 { // Clean up workaround annotations (if they exist) - annotationKeys := []string{cpuKey, memoryKey, gpuKey, labelsKey, taintsKey} + annotationKeys := []string{cpuKey, memoryKey, gpuKey, capiCPUKey, capiMemoryKey, capiGPUCountKey, capiGPUTypeKey, labelsKey, taintsKey} for _, key := range annotationKeys { delete(annotations, key) } @@ -120,15 +128,21 @@ func setScaleFromZeroAnnotationsOnObject(ctx context.Context, provider instancet } // 5. Set workaround annotations - annotations[cpuKey] = strconv.FormatInt(int64(instanceInfo.VCPU), 10) + cpuStr := strconv.FormatInt(int64(instanceInfo.VCPU), 10) + annotations[cpuKey] = cpuStr + annotations[capiCPUKey] = cpuStr annotations[memoryKey] = strconv.FormatInt(instanceInfo.MemoryMb, 10) + annotations[capiMemoryKey] = fmt.Sprintf("%dMi", instanceInfo.MemoryMb) - // Set GPU annotation only if GPU > 0 (consistent with taints handling) if instanceInfo.GPU > 0 { - annotations[gpuKey] = strconv.FormatInt(int64(instanceInfo.GPU), 10) + gpuStr := strconv.FormatInt(int64(instanceInfo.GPU), 10) + annotations[gpuKey] = gpuStr + annotations[capiGPUCountKey] = gpuStr + annotations[capiGPUTypeKey] = defaultGPUResourceName } else { - // Remove GPU annotation if there are no GPUs delete(annotations, gpuKey) + delete(annotations, capiGPUCountKey) + delete(annotations, capiGPUTypeKey) } // 6. Set labels (including architecture and NodePool.Spec.NodeLabels) diff --git a/hypershift-operator/controllers/nodepool/scale_from_zero_test.go b/hypershift-operator/controllers/nodepool/scale_from_zero_test.go index ae16ff360607..d40549b8cb63 100644 --- a/hypershift-operator/controllers/nodepool/scale_from_zero_test.go +++ b/hypershift-operator/controllers/nodepool/scale_from_zero_test.go @@ -148,6 +148,10 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { cpuKey: "4", memoryKey: "16384", gpuKey: "1", + capiCPUKey: "4", + capiMemoryKey: "16384Mi", + capiGPUCountKey: "1", + capiGPUTypeKey: "nvidia.com/gpu", labelsKey: "kubernetes.io/arch=amd64", taintsKey: "dedicated=gpu:NoSchedule", "custom.io/keep": "preserved", @@ -169,7 +173,7 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { expectErr: false, validate: func(g Gomega, md *capiv1.MachineDeployment) { a := md.GetAnnotations() - for _, k := range []string{cpuKey, memoryKey, gpuKey, labelsKey, taintsKey} { + for _, k := range []string{cpuKey, memoryKey, gpuKey, capiCPUKey, capiMemoryKey, capiGPUCountKey, capiGPUTypeKey, labelsKey, taintsKey} { g.Expect(a).ToNot(HaveKey(k)) } g.Expect(a).To(HaveKeyWithValue("custom.io/keep", "preserved")) @@ -195,8 +199,9 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { object: &capiv1.MachineDeployment{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - gpuKey: "2", - taintsKey: "old=stale:NoSchedule", + gpuKey: "2", + capiGPUCountKey: "2", + taintsKey: "old=stale:NoSchedule", }, }, }, @@ -205,9 +210,13 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { validate: func(g Gomega, md *capiv1.MachineDeployment) { a := md.GetAnnotations() g.Expect(a).To(HaveKeyWithValue(cpuKey, "2")) + g.Expect(a).To(HaveKeyWithValue(capiCPUKey, "2")) g.Expect(a).To(HaveKeyWithValue(memoryKey, "8192")) + g.Expect(a).To(HaveKeyWithValue(capiMemoryKey, "8192Mi")) g.Expect(a).To(HaveKeyWithValue(labelsKey, "kubernetes.io/arch=amd64")) g.Expect(a).ToNot(HaveKey(gpuKey)) + g.Expect(a).ToNot(HaveKey(capiGPUCountKey)) + g.Expect(a).ToNot(HaveKey(capiGPUTypeKey)) g.Expect(a).ToNot(HaveKey(taintsKey)) }, }, @@ -223,9 +232,13 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { validate: func(g Gomega, md *capiv1.MachineDeployment) { a := md.GetAnnotations() g.Expect(a).To(HaveKeyWithValue(cpuKey, "4")) + g.Expect(a).To(HaveKeyWithValue(capiCPUKey, "4")) g.Expect(a).To(HaveKeyWithValue(memoryKey, "16384")) + g.Expect(a).To(HaveKeyWithValue(capiMemoryKey, "16384Mi")) g.Expect(a).To(HaveKeyWithValue(labelsKey, "kubernetes.io/arch=amd64")) g.Expect(a).ToNot(HaveKey(gpuKey)) + g.Expect(a).ToNot(HaveKey(capiGPUCountKey)) + g.Expect(a).ToNot(HaveKey(capiGPUTypeKey)) }, }, { @@ -266,8 +279,12 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { validate: func(g Gomega, md *capiv1.MachineDeployment) { a := md.GetAnnotations() g.Expect(a).To(HaveKeyWithValue(cpuKey, "6")) + g.Expect(a).To(HaveKeyWithValue(capiCPUKey, "6")) g.Expect(a).To(HaveKeyWithValue(memoryKey, "114688")) + g.Expect(a).To(HaveKeyWithValue(capiMemoryKey, "114688Mi")) g.Expect(a).To(HaveKeyWithValue(gpuKey, "1")) + g.Expect(a).To(HaveKeyWithValue(capiGPUCountKey, "1")) + g.Expect(a).To(HaveKeyWithValue(capiGPUTypeKey, "nvidia.com/gpu")) g.Expect(a).To(HaveKeyWithValue(taintsKey, "dedicated=gpu:NoSchedule")) }, }, @@ -297,8 +314,12 @@ func TestSetScaleFromZeroAnnotationsOnObject(t *testing.T) { validate: func(g Gomega, md *capiv1.MachineDeployment) { a := md.GetAnnotations() g.Expect(a).To(HaveKeyWithValue(cpuKey, "8")) + g.Expect(a).To(HaveKeyWithValue(capiCPUKey, "8")) g.Expect(a).To(HaveKeyWithValue(memoryKey, "61440")) + g.Expect(a).To(HaveKeyWithValue(capiMemoryKey, "61440Mi")) g.Expect(a).To(HaveKeyWithValue(gpuKey, "1")) + g.Expect(a).To(HaveKeyWithValue(capiGPUCountKey, "1")) + g.Expect(a).To(HaveKeyWithValue(capiGPUTypeKey, "nvidia.com/gpu")) g.Expect(a).To(HaveKeyWithValue(labelsKey, "env=production,kubernetes.io/arch=arm64")) g.Expect(a).To(HaveKeyWithValue(taintsKey, "dedicated=gpu:NoSchedule")) g.Expect(a).To(HaveKeyWithValue("custom.io/keep", "preserved")) diff --git a/test/e2e/autoscaling_test.go b/test/e2e/autoscaling_test.go index 270af8cd9919..5fad39e05d7c 100644 --- a/test/e2e/autoscaling_test.go +++ b/test/e2e/autoscaling_test.go @@ -801,15 +801,22 @@ func testScaleFromZero(ctx context.Context, mgtClient crclient.Client, hostedClu return true, "native Status.Capacity present with CPU and Memory", nil } - // Fall back to checking annotations (pre-CAPI 1.11) + // Fall back to checking annotations t.Logf("No Status.Capacity found, checking for workaround annotations on MachineDeployment") + // CAPI autoscaler keys (--cloud-provider=clusterapi) + if _, ok := md.Annotations["capacity.cluster-autoscaler.kubernetes.io/cpu"]; !ok { + return false, "missing CAPI cpu annotation", nil + } + if _, ok := md.Annotations["capacity.cluster-autoscaler.kubernetes.io/memory"]; !ok { + return false, "missing CAPI memory annotation", nil + } + // Legacy keys (co-existing for backward compatibility) if _, ok := md.Annotations["machine.openshift.io/vCPU"]; !ok { - return false, "missing both Status.Capacity and vCPU annotation", nil + return false, "missing legacy vCPU annotation", nil } if _, ok := md.Annotations["machine.openshift.io/memoryMb"]; !ok { - return false, "missing both Status.Capacity and memoryMb annotation", nil + return false, "missing legacy memoryMb annotation", nil } - // GPU annotation is optional - only set when instance type has GPUs labels, ok := md.Annotations["capacity.cluster-autoscaler.kubernetes.io/labels"] if !ok { return false, "missing capacity labels annotation", nil @@ -844,15 +851,18 @@ func testScaleFromZero(ctx context.Context, mgtClient crclient.Client, hostedClu memQty.String(), gpuQty.String()) } else { - gpuValue := md.Annotations["machine.openshift.io/GPU"] + gpuValue := md.Annotations["capacity.cluster-autoscaler.kubernetes.io/gpu-count"] if gpuValue == "" { gpuValue = "none (non-GPU instance)" } - t.Logf("Capacity via %s: vCPU=%s, memoryMb=%s, GPU=%s, labels=%s", + t.Logf("Capacity via %s: CAPI[cpu=%s, memory=%s, gpu=%s] legacy[vCPU=%s, memoryMb=%s, GPU=%s] labels=%s", capacitySource, + md.Annotations["capacity.cluster-autoscaler.kubernetes.io/cpu"], + md.Annotations["capacity.cluster-autoscaler.kubernetes.io/memory"], + gpuValue, md.Annotations["machine.openshift.io/vCPU"], md.Annotations["machine.openshift.io/memoryMb"], - gpuValue, + md.Annotations["machine.openshift.io/GPU"], md.Annotations["capacity.cluster-autoscaler.kubernetes.io/labels"]) }