diff --git a/api/orchestration/v1alpha1/stormservice_types.go b/api/orchestration/v1alpha1/stormservice_types.go index 199a4232e..edcd8451e 100644 --- a/api/orchestration/v1alpha1/stormservice_types.go +++ b/api/orchestration/v1alpha1/stormservice_types.go @@ -62,6 +62,15 @@ type StormServiceSpec struct { // +optional Paused bool `json:"paused,omitempty" protobuf:"varint,7,opt,name=paused"` + // The maximum time in seconds for a StormService to make progress before it + // is considered to be failed. The controller will continue to process failed + // StormServices and surface a condition with a ProgressDeadlineExceeded reason. + // Progress is not estimated while a StormService is paused. Defaults to 600s. + // +kubebuilder:default=600 + // +kubebuilder:validation:Minimum=1 + // +optional + ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty" protobuf:"varint,9,opt,name=progressDeadlineSeconds"` + // DisruptionTolerance indicates how many roleSets can be unavailable during the preemption/eviction. // +optional DisruptionTolerance DisruptionTolerance `json:"disruptionTolerance,omitempty"` diff --git a/api/orchestration/v1alpha1/zz_generated.deepcopy.go b/api/orchestration/v1alpha1/zz_generated.deepcopy.go index 4c753eda8..e8bf2f3a4 100644 --- a/api/orchestration/v1alpha1/zz_generated.deepcopy.go +++ b/api/orchestration/v1alpha1/zz_generated.deepcopy.go @@ -1063,6 +1063,11 @@ func (in *StormServiceSpec) DeepCopyInto(out *StormServiceSpec) { *out = new(int32) **out = **in } + if in.ProgressDeadlineSeconds != nil { + in, out := &in.ProgressDeadlineSeconds, &out.ProgressDeadlineSeconds + *out = new(int32) + **out = **in + } in.DisruptionTolerance.DeepCopyInto(&out.DisruptionTolerance) } diff --git a/config/crd/orchestration/orchestration.aibrix.ai_stormservices.yaml b/config/crd/orchestration/orchestration.aibrix.ai_stormservices.yaml index 1b7385d15..3e97e8e35 100644 --- a/config/crd/orchestration/orchestration.aibrix.ai_stormservices.yaml +++ b/config/crd/orchestration/orchestration.aibrix.ai_stormservices.yaml @@ -58,6 +58,11 @@ spec: type: string paused: type: boolean + progressDeadlineSeconds: + default: 600 + format: int32 + minimum: 1 + type: integer replicas: format: int32 type: integer diff --git a/dist/chart/crds/orchestration.aibrix.ai_stormservices.yaml b/dist/chart/crds/orchestration.aibrix.ai_stormservices.yaml index 1b7385d15..3e97e8e35 100644 --- a/dist/chart/crds/orchestration.aibrix.ai_stormservices.yaml +++ b/dist/chart/crds/orchestration.aibrix.ai_stormservices.yaml @@ -58,6 +58,11 @@ spec: type: string paused: type: boolean + progressDeadlineSeconds: + default: 600 + format: int32 + minimum: 1 + type: integer replicas: format: int32 type: integer diff --git a/pkg/client/applyconfiguration/orchestration/v1alpha1/stormservicespec.go b/pkg/client/applyconfiguration/orchestration/v1alpha1/stormservicespec.go index 1089487e1..999c875d4 100644 --- a/pkg/client/applyconfiguration/orchestration/v1alpha1/stormservicespec.go +++ b/pkg/client/applyconfiguration/orchestration/v1alpha1/stormservicespec.go @@ -25,15 +25,16 @@ import ( // StormServiceSpecApplyConfiguration represents a declarative configuration of the StormServiceSpec type for use // with apply. type StormServiceSpecApplyConfiguration struct { - Replicas *int32 `json:"replicas,omitempty"` - Mode *v1alpha1.StormServiceMode `json:"mode,omitempty"` - Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` - Stateful *bool `json:"stateful,omitempty"` - Template *RoleSetTemplateSpecApplyConfiguration `json:"template,omitempty"` - UpdateStrategy *StormServiceUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` - RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` - Paused *bool `json:"paused,omitempty"` - DisruptionTolerance *DisruptionToleranceApplyConfiguration `json:"disruptionTolerance,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Mode *v1alpha1.StormServiceMode `json:"mode,omitempty"` + Selector *v1.LabelSelectorApplyConfiguration `json:"selector,omitempty"` + Stateful *bool `json:"stateful,omitempty"` + Template *RoleSetTemplateSpecApplyConfiguration `json:"template,omitempty"` + UpdateStrategy *StormServiceUpdateStrategyApplyConfiguration `json:"updateStrategy,omitempty"` + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + Paused *bool `json:"paused,omitempty"` + ProgressDeadlineSeconds *int32 `json:"progressDeadlineSeconds,omitempty"` + DisruptionTolerance *DisruptionToleranceApplyConfiguration `json:"disruptionTolerance,omitempty"` } // StormServiceSpecApplyConfiguration constructs a declarative configuration of the StormServiceSpec type for use with @@ -106,6 +107,14 @@ func (b *StormServiceSpecApplyConfiguration) WithPaused(value bool) *StormServic return b } +// WithProgressDeadlineSeconds sets the ProgressDeadlineSeconds field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ProgressDeadlineSeconds field is set to the value of the last call. +func (b *StormServiceSpecApplyConfiguration) WithProgressDeadlineSeconds(value int32) *StormServiceSpecApplyConfiguration { + b.ProgressDeadlineSeconds = &value + return b +} + // WithDisruptionTolerance sets the DisruptionTolerance field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the DisruptionTolerance field is set to the value of the last call. diff --git a/pkg/controller/stormservice/progress.go b/pkg/controller/stormservice/progress.go new file mode 100644 index 000000000..77b4cc858 --- /dev/null +++ b/pkg/controller/stormservice/progress.go @@ -0,0 +1,147 @@ +/* +Copyright 2026 The Aibrix Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stormservice + +import ( + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + orchestrationv1alpha1 "github.com/vllm-project/aibrix/api/orchestration/v1alpha1" + utils "github.com/vllm-project/aibrix/pkg/controller/util/orchestration" +) + +const ( + defaultProgressDeadlineSeconds int32 = 600 + + ProgressingReason = "Processing" + ProgressDeadlineExceededReason = "ProgressDeadlineExceeded" + PausedReason = "DeploymentPaused" + ResumedReason = "DeploymentResumed" +) + +func newProgressingCondition(status corev1.ConditionStatus, reason, message string, now time.Time) orchestrationv1alpha1.Condition { + timestamp := metav1.NewTime(now) + return orchestrationv1alpha1.Condition{ + Type: orchestrationv1alpha1.StormServiceProgressing, + Status: status, + LastUpdateTime: ×tamp, + LastTransitionTime: ×tamp, + Reason: reason, + Message: message, + } +} + +func stormServiceProgressing(oldStatus, newStatus *orchestrationv1alpha1.StormServiceStatus) bool { + oldReplicas := oldStatus.Replicas - oldStatus.UpdatedReplicas + newReplicas := newStatus.Replicas - newStatus.UpdatedReplicas + return newStatus.UpdatedReplicas > oldStatus.UpdatedReplicas || + newReplicas < oldReplicas || + newStatus.ReadyReplicas > oldStatus.ReadyReplicas || + newStatus.UpdatedReadyReplicas > oldStatus.UpdatedReadyReplicas +} + +func progressDeadline(stormService *orchestrationv1alpha1.StormService) time.Duration { + seconds := defaultProgressDeadlineSeconds + if stormService.Spec.ProgressDeadlineSeconds != nil { + seconds = *stormService.Spec.ProgressDeadlineSeconds + } + return time.Duration(seconds) * time.Second +} + +func syncStormServiceProgressingCondition(stormService *orchestrationv1alpha1.StormService, oldStatus *orchestrationv1alpha1.StormServiceStatus, now time.Time) { + currentCondition := utils.GetCondition(oldStatus.Conditions, orchestrationv1alpha1.StormServiceProgressing) + var condition orchestrationv1alpha1.Condition + + switch { + case stormService.Spec.Paused: + if currentCondition != nil && currentCondition.Reason == ProgressDeadlineExceededReason { + condition = *currentCondition + } else if currentCondition != nil && currentCondition.Reason == PausedReason { + condition = *currentCondition + } else { + condition = newProgressingCondition( + corev1.ConditionUnknown, + PausedReason, + fmt.Sprintf("StormService %q is paused.", stormService.Name), + now, + ) + } + case currentCondition != nil && currentCondition.Reason == PausedReason: + condition = newProgressingCondition( + corev1.ConditionUnknown, + ResumedReason, + fmt.Sprintf("StormService %q is resumed.", stormService.Name), + now, + ) + condition.LastTransitionTime = currentCondition.LastTransitionTime + case stormServiceProgressing(oldStatus, &stormService.Status): + condition = newProgressingCondition( + corev1.ConditionTrue, + ProgressingReason, + fmt.Sprintf("StormService %q is progressing.", stormService.Name), + now, + ) + if currentCondition != nil && currentCondition.Status == corev1.ConditionTrue { + condition.LastTransitionTime = currentCondition.LastTransitionTime + } + case currentCondition == nil || currentCondition.LastUpdateTime == nil: + condition = newProgressingCondition( + corev1.ConditionTrue, + ProgressingReason, + fmt.Sprintf("StormService %q is progressing.", stormService.Name), + now, + ) + case currentCondition.Reason == ProgressDeadlineExceededReason: + condition = *currentCondition + case !currentCondition.LastUpdateTime.Add(progressDeadline(stormService)).After(now): + condition = newProgressingCondition( + corev1.ConditionFalse, + ProgressDeadlineExceededReason, + fmt.Sprintf("StormService %q has timed out progressing.", stormService.Name), + now, + ) + default: + condition = *currentCondition + } + + stormService.Status.Conditions = append( + utils.FilterOutCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing), + condition, + ) +} + +func progressDeadlineRequeueAfter(stormService *orchestrationv1alpha1.StormService, now time.Time) time.Duration { + if stormService.Spec.Paused { + return DefaultRequeueAfter + } + condition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + if condition == nil || condition.LastUpdateTime == nil || condition.Reason == ProgressDeadlineExceededReason { + return DefaultRequeueAfter + } + + requeueAfter := condition.LastUpdateTime.Add(progressDeadline(stormService)).Sub(now) + time.Second + if requeueAfter < time.Second { + return time.Second + } + if requeueAfter < DefaultRequeueAfter { + return requeueAfter + } + return DefaultRequeueAfter +} diff --git a/pkg/controller/stormservice/progress_test.go b/pkg/controller/stormservice/progress_test.go new file mode 100644 index 000000000..c47239262 --- /dev/null +++ b/pkg/controller/stormservice/progress_test.go @@ -0,0 +1,261 @@ +/* +Copyright 2026 The Aibrix Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stormservice + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + orchestrationv1alpha1 "github.com/vllm-project/aibrix/api/orchestration/v1alpha1" + utils "github.com/vllm-project/aibrix/pkg/controller/util/orchestration" +) + +func progressingCondition(status corev1.ConditionStatus, reason string, updated, transitioned time.Time) orchestrationv1alpha1.Condition { + lastUpdate := metav1.NewTime(updated) + lastTransition := metav1.NewTime(transitioned) + return orchestrationv1alpha1.Condition{ + Type: orchestrationv1alpha1.StormServiceProgressing, + Status: status, + Reason: reason, + LastUpdateTime: &lastUpdate, + LastTransitionTime: &lastTransition, + } +} + +func TestSyncStormServiceProgressingConditionTimesOutStalledRollout(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + now := started.Add(11 * time.Second) + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Replicas: 2, + UpdatedReplicas: 1, + ReadyReplicas: 1, + UpdatedReadyReplicas: 1, + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, started), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "stalled"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + + syncStormServiceProgressingCondition(stormService, &oldStatus, now) + + condition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, condition) + assert.Equal(t, corev1.ConditionFalse, condition.Status) + assert.Equal(t, ProgressDeadlineExceededReason, condition.Reason) + assert.Equal(t, now, condition.LastUpdateTime.Time) + assert.Equal(t, now, condition.LastTransitionTime.Time) +} + +func TestSyncStormServiceProgressingConditionRefreshesOnlyOnProgress(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + now := started.Add(9 * time.Second) + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Replicas: 2, + UpdatedReplicas: 1, + ReadyReplicas: 1, + UpdatedReadyReplicas: 1, + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, started.Add(-time.Minute)), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "progressing"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + stormService.Status.UpdatedReplicas = 2 + + syncStormServiceProgressingCondition(stormService, &oldStatus, now) + + condition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, condition) + assert.Equal(t, corev1.ConditionTrue, condition.Status) + assert.Equal(t, ProgressingReason, condition.Reason) + assert.Equal(t, now, condition.LastUpdateTime.Time) + assert.Equal(t, started.Add(-time.Minute), condition.LastTransitionTime.Time) +} + +func TestSyncStormServiceProgressingConditionPreservesOtherConditions(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + now := started.Add(9 * time.Second) + readyUpdated := metav1.NewTime(started.Add(-2 * time.Minute)) + readyTransitioned := metav1.NewTime(started.Add(-3 * time.Minute)) + readyCondition := orchestrationv1alpha1.Condition{ + Type: orchestrationv1alpha1.StormServiceReady, + Status: corev1.ConditionFalse, + Reason: "MinimumReplicasUnavailable", + Message: "StormService does not have minimum availability.", + LastUpdateTime: &readyUpdated, + LastTransitionTime: &readyTransitioned, + } + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Replicas: 2, + UpdatedReplicas: 1, + ReadyReplicas: 1, + UpdatedReadyReplicas: 1, + Conditions: orchestrationv1alpha1.Conditions{ + readyCondition, + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, started.Add(-time.Minute)), + // A malformed duplicate must not survive the type-specific replacement. + progressingCondition(corev1.ConditionTrue, ProgressingReason, started.Add(-time.Second), started.Add(-time.Minute)), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "progressing"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + stormService.Status.UpdatedReplicas = 2 + + syncStormServiceProgressingCondition(stormService, &oldStatus, now) + + ready := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceReady) + require.NotNil(t, ready) + assert.Equal(t, readyCondition, *ready) + progressing := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, progressing) + assert.Equal(t, now, progressing.LastUpdateTime.Time) + assert.Len(t, stormService.Status.Conditions, 2) +} + +func TestSyncStormServiceProgressingConditionDoesNotRefreshWithoutProgress(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + transitioned := started.Add(-time.Minute) + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Replicas: 2, + UpdatedReplicas: 1, + ReadyReplicas: 1, + UpdatedReadyReplicas: 1, + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, transitioned), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "stalled-within-deadline"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + + syncStormServiceProgressingCondition(stormService, &oldStatus, started.Add(9*time.Second)) + + condition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, condition) + assert.Equal(t, corev1.ConditionTrue, condition.Status) + assert.Equal(t, ProgressingReason, condition.Reason) + assert.Equal(t, started, condition.LastUpdateTime.Time) + assert.Equal(t, transitioned, condition.LastTransitionTime.Time) +} + +func TestSyncStormServiceProgressingConditionExcludesPausedTime(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, started), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "paused"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + Paused: true, + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + pausedAt := started.Add(time.Hour) + + syncStormServiceProgressingCondition(stormService, &oldStatus, pausedAt) + pausedCondition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, pausedCondition) + assert.Equal(t, corev1.ConditionUnknown, pausedCondition.Status) + assert.Equal(t, PausedReason, pausedCondition.Reason) + + pausedStatus := stormService.Status.DeepCopy() + stormService.Spec.Paused = false + resumedAt := pausedAt.Add(time.Hour) + syncStormServiceProgressingCondition(stormService, pausedStatus, resumedAt) + resumedCondition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, resumedCondition) + assert.Equal(t, corev1.ConditionUnknown, resumedCondition.Status) + assert.Equal(t, ResumedReason, resumedCondition.Reason) + assert.Equal(t, resumedAt, resumedCondition.LastUpdateTime.Time) + assert.Equal(t, pausedAt, resumedCondition.LastTransitionTime.Time) +} + +func TestSyncStormServiceProgressingConditionRecoversAfterProgress(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + now := started.Add(time.Minute) + oldStatus := orchestrationv1alpha1.StormServiceStatus{ + Replicas: 2, + UpdatedReplicas: 1, + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionFalse, ProgressDeadlineExceededReason, started, started), + }, + } + stormService := &orchestrationv1alpha1.StormService{ + ObjectMeta: metav1.ObjectMeta{Name: "recovered"}, + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: *oldStatus.DeepCopy(), + } + stormService.Status.UpdatedReplicas = 2 + + syncStormServiceProgressingCondition(stormService, &oldStatus, now) + + condition := utils.GetCondition(stormService.Status.Conditions, orchestrationv1alpha1.StormServiceProgressing) + require.NotNil(t, condition) + assert.Equal(t, corev1.ConditionTrue, condition.Status) + assert.Equal(t, ProgressingReason, condition.Reason) + assert.Equal(t, now, condition.LastUpdateTime.Time) +} + +func TestProgressDeadlineRequeueAfter(t *testing.T) { + started := time.Date(2026, 8, 23, 1, 2, 3, 0, time.UTC) + stormService := &orchestrationv1alpha1.StormService{ + Spec: orchestrationv1alpha1.StormServiceSpec{ + ProgressDeadlineSeconds: ptr.To[int32](10), + }, + Status: orchestrationv1alpha1.StormServiceStatus{ + Conditions: orchestrationv1alpha1.Conditions{ + progressingCondition(corev1.ConditionTrue, ProgressingReason, started, started), + }, + }, + } + + assert.Equal(t, 8*time.Second, progressDeadlineRequeueAfter(stormService, started.Add(3*time.Second))) + + stormService.Spec.Paused = true + assert.Equal(t, DefaultRequeueAfter, progressDeadlineRequeueAfter(stormService, started.Add(3*time.Second))) +} diff --git a/pkg/controller/stormservice/sync.go b/pkg/controller/stormservice/sync.go index 731ed3daa..56d24e03e 100644 --- a/pkg/controller/stormservice/sync.go +++ b/pkg/controller/stormservice/sync.go @@ -65,7 +65,7 @@ func (r *StormServiceReconciler) sync(ctx context.Context, stormService *orchest klog.Errorf("failed to update status for stormservice %s/%s, err: %v", stormService.Namespace, stormService.Name, err) return 0, err } else if !ready { - return DefaultRequeueAfter, nil + return progressDeadlineRequeueAfter(stormService, time.Now()), nil } return 0, nil } @@ -405,9 +405,7 @@ func (r *StormServiceReconciler) updateStatus(ctx context.Context, stormService *utils.NewCondition(orchestrationv1alpha1.StormServiceReady, corev1.ConditionTrue, "Ready", ""), } } else { - stormService.Status.Conditions = []orchestrationv1alpha1.Condition{ - *utils.NewCondition(orchestrationv1alpha1.StormServiceProgressing, corev1.ConditionTrue, "Processing", ""), - } + syncStormServiceProgressingCondition(stormService, checkpoint, time.Now()) } // support scale sub resources. // TODO: add pod template hash to avoid errors during upgrade. diff --git a/test/integration/controller/stormservice_test.go b/test/integration/controller/stormservice_test.go index 17598e624..581da2da2 100644 --- a/test/integration/controller/stormservice_test.go +++ b/test/integration/controller/stormservice_test.go @@ -62,6 +62,26 @@ var _ = ginkgo.Describe("StormService controller test", func() { gomega.Expect(k8sClient.Delete(ctx, ns)).To(gomega.Succeed()) }) + makeProgressDeadlineStormService := func(name string, replicas int32) *orchestrationapi.StormService { + matchLabel := map[string]string{"app": name} + roleSetSpec := &orchestrationapi.RoleSetSpec{ + Roles: []orchestrationapi.RoleSpec{ + { + Name: "vllm", + Replicas: ptr.To(int32(1)), + Template: validation.MakePodTemplate("vllm-openai:v0.10.0-cu128-nixl-v0.4.1-lmcache-0.3.2"), + }, + }, + } + return wrapper.MakeStormService(name). + Namespace(ns.Name). + Replicas(ptr.To(replicas)). + Selector(metav1.SetAsLabelSelector(matchLabel)). + UpdateStrategyType(orchestrationapi.RollingUpdateStormServiceStrategyType). + RoleSetTemplateMeta(metav1.ObjectMeta{Labels: matchLabel}, roleSetSpec). + Obj() + } + // testValidatingCase defines a test case with initial setup and a series of updates type testValidatingCase struct { makeStormService func() *orchestrationapi.StormService @@ -390,6 +410,188 @@ var _ = ginkgo.Describe("StormService controller test", func() { markPodReadyWithRuntimeImage(ctx, k8sClient, replacementPod, prefillImageVersionV1) validation.ValidateStormServiceStatus(ctx, k8sClient, ss, 1, 1, 0, 1, 1, 1, true) }) + + ginkgo.It("defaults the progress deadline and refreshes it when a vLLM rollout progresses", func() { + ss := makeProgressDeadlineStormService("stormservice-progress-refresh", 2) + gomega.Expect(k8sClient.Create(ctx, ss)).To(gomega.Succeed()) + validation.WaitForRoleSetsCreated(ctx, k8sClient, ns.Name, ss.Name, 2) + validation.WaitForPodsCreated(ctx, k8sClient, ns.Name, constants.StormServiceNameLabelKey, ss.Name, 2) + + var progressStartedAt time.Time + gomega.Eventually(func(g gomega.Gomega) { + latest := &orchestrationapi.StormService{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest)).To(gomega.Succeed()) + g.Expect(latest.Spec.ProgressDeadlineSeconds).NotTo(gomega.BeNil()) + g.Expect(*latest.Spec.ProgressDeadlineSeconds).To(gomega.Equal(int32(600))) + condition := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Status).To(gomega.Equal(corev1.ConditionTrue)) + g.Expect(condition.LastUpdateTime).NotTo(gomega.BeNil()) + progressStartedAt = condition.LastUpdateTime.Time + }, time.Second*10, time.Millisecond*100).Should(gomega.Succeed()) + + if wait := time.Until(progressStartedAt.Add(time.Second)); wait > 0 { + time.Sleep(wait + 100*time.Millisecond) + } + roleSets := listStormServiceRoleSets(ctx, k8sClient, ns.Name, ss.Name) + gomega.Expect(roleSets).To(gomega.HaveLen(2)) + markRoleSetPodsReady(ctx, k8sClient, ns.Name, roleSets[0].Name) + waitForRoleSetReady(ctx, k8sClient, ns.Name, roleSets[0].Name) + + gomega.Eventually(func(g gomega.Gomega) { + latest := &orchestrationapi.StormService{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest)).To(gomega.Succeed()) + condition := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Status).To(gomega.Equal(corev1.ConditionTrue)) + g.Expect(latest.Status.ReadyReplicas).To(gomega.Equal(int32(1))) + g.Expect(condition.LastUpdateTime).NotTo(gomega.BeNil()) + g.Expect(condition.LastUpdateTime.Time).To(gomega.BeTemporally(">", progressStartedAt)) + }, time.Second*10, time.Millisecond*100).Should(gomega.Succeed()) + }) + + ginkgo.It("preserves other conditions while updating rollout progress", func() { + ss := makeProgressDeadlineStormService("stormservice-progress-preserve-conditions", 1) + ss.Spec.Paused = true + + gomega.Expect(k8sClient.Create(ctx, ss)).To(gomega.Succeed()) + validation.WaitForRoleSetsCreated(ctx, k8sClient, ns.Name, ss.Name, 1) + validation.WaitForPodsCreated(ctx, k8sClient, ns.Name, constants.StormServiceNameLabelKey, ss.Name, 1) + + gomega.Eventually(func(g gomega.Gomega) { + latest := &orchestrationapi.StormService{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest)).To(gomega.Succeed()) + progressing := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + g.Expect(progressing).NotTo(gomega.BeNil()) + g.Expect(progressing.Reason).To(gomega.Equal("DeploymentPaused")) + }, time.Second*10, time.Millisecond*100).Should(gomega.Succeed()) + + readyUpdated := metav1.NewTime(time.Date(2026, 8, 26, 1, 2, 3, 0, time.UTC)) + readyCondition := orchestrationapi.Condition{ + Type: orchestrationapi.StormServiceReady, + Status: corev1.ConditionFalse, + Reason: "MinimumReplicasUnavailable", + Message: "StormService does not have minimum availability.", + LastUpdateTime: &readyUpdated, + LastTransitionTime: &readyUpdated, + } + gomega.Eventually(func() error { + latest := &orchestrationapi.StormService{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest); err != nil { + return err + } + conditions := make(orchestrationapi.Conditions, 0, len(latest.Status.Conditions)+1) + for _, condition := range latest.Status.Conditions { + if condition.Type != orchestrationapi.StormServiceReady { + conditions = append(conditions, condition) + } + } + latest.Status.Conditions = append(conditions, readyCondition) + if err := k8sClient.Status().Update(ctx, latest); err != nil { + return err + } + latest.Spec.Paused = false + return k8sClient.Update(ctx, latest) + }, time.Second*10, time.Millisecond*100).Should(gomega.Succeed()) + + gomega.Eventually(func(g gomega.Gomega) { + latest := &orchestrationapi.StormService{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest)).To(gomega.Succeed()) + progressing := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + g.Expect(progressing).NotTo(gomega.BeNil()) + g.Expect(progressing.Reason).To(gomega.Equal("DeploymentResumed")) + ready := validation.FindCondition(string(orchestrationapi.StormServiceReady), latest.Status.Conditions) + g.Expect(ready).NotTo(gomega.BeNil()) + g.Expect(ready.Type).To(gomega.Equal(readyCondition.Type)) + g.Expect(ready.Status).To(gomega.Equal(readyCondition.Status)) + g.Expect(ready.Reason).To(gomega.Equal(readyCondition.Reason)) + g.Expect(ready.Message).To(gomega.Equal(readyCondition.Message)) + g.Expect(ready.LastUpdateTime.Time).To(gomega.BeTemporally("==", readyCondition.LastUpdateTime.Time)) + g.Expect(ready.LastTransitionTime.Time).To(gomega.BeTemporally("==", readyCondition.LastTransitionTime.Time)) + g.Expect(ready.LastUpdateMicroTime).To(gomega.BeNil()) + }, time.Second*10, time.Millisecond*100).Should(gomega.Succeed()) + }) + + ginkgo.It("excludes time spent paused from the progress deadline", func() { + ss := makeProgressDeadlineStormService("stormservice-progress-paused", 1) + ss.Spec.Paused = true + ss.Spec.ProgressDeadlineSeconds = ptr.To(int32(2)) + + gomega.Expect(k8sClient.Create(ctx, ss)).To(gomega.Succeed()) + validation.WaitForRoleSetsCreated(ctx, k8sClient, ns.Name, ss.Name, 1) + validation.WaitForPodsCreated(ctx, k8sClient, ns.Name, constants.StormServiceNameLabelKey, ss.Name, 1) + + gomega.Eventually(func() bool { + latest := &orchestrationapi.StormService{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest); err != nil { + return false + } + condition := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + return condition != nil && condition.Status == corev1.ConditionUnknown && condition.Reason == "DeploymentPaused" + }, time.Second*10, time.Millisecond*100).Should(gomega.BeTrue()) + + gomega.Consistently(func() bool { + latest := &orchestrationapi.StormService{} + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest); err != nil { + return false + } + condition := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + return condition != nil && condition.Status == corev1.ConditionUnknown && condition.Reason == "DeploymentPaused" + }, 3*time.Second, time.Millisecond*100).Should(gomega.BeTrue()) + }) + + ginkgo.It("reports a stalled vLLM rollout deadline and recovers without replacing the RoleSet", func() { + ss := makeProgressDeadlineStormService("stormservice-progress-deadline", 1) + ss.Spec.ProgressDeadlineSeconds = ptr.To(int32(2)) + + startedAt := time.Now() + gomega.Expect(k8sClient.Create(ctx, ss)).To(gomega.Succeed()) + validation.WaitForRoleSetsCreated(ctx, k8sClient, ns.Name, ss.Name, 1) + validation.WaitForPodsCreated(ctx, k8sClient, ns.Name, constants.StormServiceNameLabelKey, ss.Name, 1) + initialRoleSets := listStormServiceRoleSets(ctx, k8sClient, ns.Name, ss.Name) + gomega.Expect(initialRoleSets).To(gomega.HaveLen(1)) + initialRoleSetUID := initialRoleSets[0].UID + + gomega.Eventually(func(g gomega.Gomega) { + latest := &orchestrationapi.StormService{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(ss), latest)).To(gomega.Succeed()) + condition := validation.FindCondition( + string(orchestrationapi.StormServiceProgressing), + latest.Status.Conditions, + ) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Status).To(gomega.Equal(corev1.ConditionFalse)) + g.Expect(condition.Reason).To(gomega.Equal("ProgressDeadlineExceeded")) + }, time.Second*15, time.Millisecond*100).Should(gomega.Succeed()) + gomega.Expect(time.Since(startedAt)).To(gomega.BeNumerically(">=", 2*time.Second)) + timedOutRoleSets := listStormServiceRoleSets(ctx, k8sClient, ns.Name, ss.Name) + gomega.Expect(timedOutRoleSets).To(gomega.HaveLen(1)) + gomega.Expect(timedOutRoleSets[0].UID).To(gomega.Equal(initialRoleSetUID)) + + validation.MarkPodsReady(ctx, k8sClient, ns.Name, constants.StormServiceNameLabelKey, ss.Name) + validation.ValidateStormServiceStatus(ctx, k8sClient, ss, 1, 1, 0, 1, 1, 1, true) + recoveredRoleSets := listStormServiceRoleSets(ctx, k8sClient, ns.Name, ss.Name) + gomega.Expect(recoveredRoleSets).To(gomega.HaveLen(1)) + gomega.Expect(recoveredRoleSets[0].UID).To(gomega.Equal(initialRoleSetUID)) + }) }) func waitForSingleStormServiceRolePod( diff --git a/test/integration/webhook/stormservice_webhook_test.go b/test/integration/webhook/stormservice_webhook_test.go index 56a2a2e0c..e38878ec2 100644 --- a/test/integration/webhook/stormservice_webhook_test.go +++ b/test/integration/webhook/stormservice_webhook_test.go @@ -70,7 +70,9 @@ var _ = ginkgo.Describe("stormservice default webhook", func() { func(tc *testDefaultingCase) { model := tc.stormservice() gomega.Expect(k8sClient.Create(ctx, model)).To(gomega.Succeed()) - gomega.Expect(model).To(gomega.BeComparableTo(tc.wantStormService(), + want := tc.wantStormService() + want.Spec.ProgressDeadlineSeconds = ptr.To(int32(600)) + gomega.Expect(model).To(gomega.BeComparableTo(want, cmpopts.IgnoreTypes(orchestrationapi.StormServiceStatus{}), cmpopts.IgnoreFields(metav1.ObjectMeta{}, "UID", "ResourceVersion", "Generation", "CreationTimestamp", "ManagedFields")), @@ -162,6 +164,32 @@ var _ = ginkgo.Describe("stormservice default webhook", func() { failed: false, }), + ginkgo.Entry("rejects zero progress deadline", &testValidatingCase{ + stormservice: func() *orchestrationapi.StormService { + stormService := wrapper.MakeStormService("zero-progress-deadline"). + Namespace(ns.Name). + WithDefaultConfiguration(). + Obj() + stormService.Spec.ProgressDeadlineSeconds = ptr.To(int32(0)) + return stormService + }, + failed: true, + expectInvalid: true, + }), + + ginkgo.Entry("rejects negative progress deadline", &testValidatingCase{ + stormservice: func() *orchestrationapi.StormService { + stormService := wrapper.MakeStormService("negative-progress-deadline"). + Namespace(ns.Name). + WithDefaultConfiguration(). + Obj() + stormService.Spec.ProgressDeadlineSeconds = ptr.To(int32(-1)) + return stormService + }, + failed: true, + expectInvalid: true, + }), + ginkgo.Entry("rejects a missing nested RoleSet spec", &testValidatingCase{ stormservice: func() *orchestrationapi.StormService { stormService := wrapper.MakeStormService("missing-nested-spec").