diff --git a/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/kpa.yaml b/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/kpa.yaml index c49d4546b..cd822e7ce 100644 --- a/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/kpa.yaml +++ b/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/kpa.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 3m + autoscaling.aibrix.ai/scale-down-cooldown-window: 3m spec: scalingStrategy: KPA minReplicas: 1 diff --git a/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/optimizer-kpa.yaml b/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/optimizer-kpa.yaml index 0d04a4e9b..28c6dadc7 100644 --- a/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/optimizer-kpa.yaml +++ b/benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/optimizer-kpa.yaml @@ -7,14 +7,14 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s spec: scalingStrategy: KPA minReplicas: 1 maxReplicas: 8 metricsSources: - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 - metricSourceType: domain + metricSourceType: external path: /metrics/default/deepseek-llm-7b-chat protocolType: http targetMetric: vllm:deployment_replicas diff --git a/config/samples/autoscaling_v1alpha1_mock_llama.yaml b/config/samples/autoscaling_v1alpha1_mock_llama.yaml index a228f0d6b..468d8a076 100644 --- a/config/samples/autoscaling_v1alpha1_mock_llama.yaml +++ b/config/samples/autoscaling_v1alpha1_mock_llama.yaml @@ -8,7 +8,7 @@ metadata: annotations: autoscaling.aibrix.ai/max-scale-up-rate: "2" autoscaling.aibrix.ai/max-scale-down-rate: "2" - kpa.autoscaling.aibrix.ai/scale-down-delay: "60s" + autoscaling.aibrix.ai/scale-down-cooldown-window: "60s" namespace: aibrix-system spec: scaleTargetRef: diff --git a/development/app/config/heterogeneous/simulator_a40/patch_podautoscaler_a40.yaml b/development/app/config/heterogeneous/simulator_a40/patch_podautoscaler_a40.yaml index a68c15663..a53fd6f7c 100644 --- a/development/app/config/heterogeneous/simulator_a40/patch_podautoscaler_a40.yaml +++ b/development/app/config/heterogeneous/simulator_a40/patch_podautoscaler_a40.yaml @@ -4,14 +4,14 @@ kind: PodAutoscaler metadata: name: podautoscaler-simulator-llama2-7b-a40 annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: simulator-llama2-7b-a40 metricsSources: - - metricSourceType: domain + - metricSourceType: external protocolType: http endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 path: /metrics/default/simulator-llama2-7b-a40 diff --git a/development/app/config/simulator/patch_podautoscaler_a100.yaml b/development/app/config/simulator/patch_podautoscaler_a100.yaml index 823e5cffc..d968ab48b 100644 --- a/development/app/config/simulator/patch_podautoscaler_a100.yaml +++ b/development/app/config/simulator/patch_podautoscaler_a100.yaml @@ -4,14 +4,14 @@ kind: PodAutoscaler metadata: name: podautoscaler-simulator-llama2-7b-a100 annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: simulator-llama2-7b-a100 metricsSources: - - metricSourceType: domain + - metricSourceType: external protocolType: http endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 path: /metrics/default/simulator-llama2-7b-a100 diff --git a/development/app/config/templates/podautoscaler/podautoscaler_kpa.yaml b/development/app/config/templates/podautoscaler/podautoscaler_kpa.yaml index 5d89bf0ee..346e96c3d 100644 --- a/development/app/config/templates/podautoscaler/podautoscaler_kpa.yaml +++ b/development/app/config/templates/podautoscaler/podautoscaler_kpa.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 30s + autoscaling.aibrix.ai/scale-down-cooldown-window: 30s namespace: default spec: scaleTargetRef: diff --git a/development/tutorials/distributed/fleet-autoscaling.yaml b/development/tutorials/distributed/fleet-autoscaling.yaml index 605bb08a8..a81e29bf2 100644 --- a/development/tutorials/distributed/fleet-autoscaling.yaml +++ b/development/tutorials/distributed/fleet-autoscaling.yaml @@ -6,7 +6,7 @@ metadata: labels: app.kubernetes.io/name: aibrix annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 1m + autoscaling.aibrix.ai/scale-down-cooldown-window: 1m spec: scalingStrategy: KPA minReplicas: 1 diff --git a/pkg/controller/podautoscaler/metrics/fetcher.go b/pkg/controller/podautoscaler/metrics/fetcher.go index bd9913ad8..fd4ff7b61 100644 --- a/pkg/controller/podautoscaler/metrics/fetcher.go +++ b/pkg/controller/podautoscaler/metrics/fetcher.go @@ -19,6 +19,7 @@ package metrics import ( "context" "fmt" + "strings" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -272,9 +273,16 @@ func (f *ExternalMetricsFetcher) fetchFromGPUOptimizer(ctx context.Context, pod "path", source.Path, "metric", source.TargetMetric) - // Use the centralized engine fetcher for external HTTP calls - // This gives us a global value that we need to adapt to per-pod semantics - metricValue, err := f.engineFetcher.FetchTypedMetric(ctx, source.Endpoint, "external", "gpu-optimizer", source.TargetMetric) + protocol := source.ProtocolType + if protocol == "" { + protocol = autoscalingv1alpha1.HTTP + } + url := fmt.Sprintf("%s://%s/%s", protocol, source.Endpoint, strings.TrimLeft(source.Path, "/")) + + // External metrics are not engine metrics: fetch the raw metric directly from the + // configured endpoint and path instead of resolving through the central registry. + // This gives us a global value that we need to adapt to per-pod semantics. + metricValue, err := f.engineFetcher.FetchRawMetric(ctx, url, source.Endpoint, source.TargetMetric) if err != nil { klog.Warningf("Failed to fetch metric %s from GPU-Optimizer %s: %v", source.TargetMetric, source.Endpoint, err) diff --git a/pkg/metrics/engine_fetcher.go b/pkg/metrics/engine_fetcher.go index 639e891ad..921c584cd 100644 --- a/pkg/metrics/engine_fetcher.go +++ b/pkg/metrics/engine_fetcher.go @@ -124,10 +124,8 @@ func (ef *EngineMetricsFetcher) FetchTypedMetric(ctx context.Context, endpoint, klog.V(4).InfoS("Retrying typed metric fetch from engine endpoint", "attempt", attempt, "delay", delay, "identifier", identifier, "metric", metricName) - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(delay): + if err := sleepWithContext(ctx, delay); err != nil { + return nil, err } } @@ -156,6 +154,48 @@ func (ef *EngineMetricsFetcher) FetchTypedMetric(ctx context.Context, endpoint, metricName, identifier, ef.config.MaxRetries+1) } +// FetchRawMetric fetches a metric by its raw Prometheus name from an explicit metrics URL, +// bypassing the central metric registry. External sources such as the GPU optimizer expose +// caller-defined metrics on caller-defined paths, so neither the registry's metric +// definitions nor its per-engine paths apply to them. +func (ef *EngineMetricsFetcher) FetchRawMetric(ctx context.Context, url, identifier, rawMetricName string) (MetricValue, error) { + for attempt := 0; attempt <= ef.config.MaxRetries; attempt++ { + if attempt > 0 { + delay := ef.calculateBackoffDelay(attempt) + klog.V(4).InfoS("Retrying raw metric fetch", + "attempt", attempt, "delay", delay, "identifier", identifier, "metric", rawMetricName) + + if err := sleepWithContext(ctx, delay); err != nil { + return nil, err + } + } + + // Do not attribute failures of external sources to the engine failure counter. + allMetrics, err := ef.fetchMetricsFromURL(ctx, url, "") + if err != nil { + klog.V(4).InfoS("Failed to fetch metrics from URL", + "attempt", attempt+1, "identifier", identifier, "url", url, "error", err) + continue + } + + family, exists := allMetrics[rawMetricName] + if !exists || len(family.Metric) == 0 { + klog.V(4).InfoS("Raw metric not found in response", + "attempt", attempt+1, "identifier", identifier, "metric", rawMetricName) + continue + } + + metricValue, err := GetCounterGaugeValue(family.Metric[0], family.GetType()) + if err != nil { + return nil, fmt.Errorf("failed to parse raw metric %s from %s: %w", rawMetricName, identifier, err) + } + return metricValue, nil + } + + return nil, fmt.Errorf("failed to fetch raw metric %s from %s after %d attempts", + rawMetricName, identifier, ef.config.MaxRetries+1) +} + // FetchAllTypedMetrics fetches all available typed metrics from an engine endpoint func (ef *EngineMetricsFetcher) FetchAllTypedMetrics(ctx context.Context, endpoint, engineType, identifier string, requestedMetrics []string) (*EngineMetricsResult, error) { result := &EngineMetricsResult{ @@ -179,10 +219,8 @@ func (ef *EngineMetricsFetcher) FetchAllTypedMetrics(ctx context.Context, endpoi klog.V(4).InfoS("Retrying all typed metrics fetch from engine endpoint", "attempt", attempt, "delay", delay, "identifier", identifier) - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(delay): + if err := sleepWithContext(ctx, delay); err != nil { + return nil, err } } @@ -276,6 +314,21 @@ func (ef *EngineMetricsFetcher) calculateBackoffDelay(attempt int) time.Duration return delay } +// sleepWithContext blocks for the given delay or until ctx is done, whichever comes first. +// It uses an explicit timer that is stopped on early return so a cancelled context does not +// leave a pending timer behind, unlike time.After, whose timer cannot be stopped. +func sleepWithContext(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + // getAvailableMetricsForEngine returns all metrics available for a given engine type func (ef *EngineMetricsFetcher) getAvailableMetricsForEngine(engineType string) []string { var availableMetrics []string @@ -423,8 +476,16 @@ func (ef *EngineMetricsFetcher) parseMetricInstance(familyMetric *dto.Metric, me return nil, fmt.Errorf("unsupported metric type for raw parsing: %v", metric.MetricType) } -// fetchAllMetricsFromURL performs a single HTTP request and parses all Prometheus metrics +// fetchAllMetricsFromURL performs a single HTTP request against an engine endpoint and parses +// all Prometheus metrics. Transport failures are counted in llm_engine_metrics_query_fail. func (ef *EngineMetricsFetcher) fetchAllMetricsFromURL(ctx context.Context, url string) (map[string]*dto.MetricFamily, error) { + return ef.fetchMetricsFromURL(ctx, url, LLMEngineMetricsQueryFail) +} + +// fetchMetricsFromURL performs a single HTTP request and parses all Prometheus metrics in the +// response. If failureMetric is non-empty, transport failures increment that counter; callers +// scraping non-engine sources pass an empty name so their failures are not attributed to engines. +func (ef *EngineMetricsFetcher) fetchMetricsFromURL(ctx context.Context, url, failureMetric string) (map[string]*dto.MetricFamily, error) { // Use our configured HTTP client with the existing parsing logic req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { @@ -433,7 +494,9 @@ func (ef *EngineMetricsFetcher) fetchAllMetricsFromURL(ctx context.Context, url resp, err := ef.client.Do(req) if err != nil { - EmitMetricToPrometheus(nil, nil, LLMEngineMetricsQueryFail, &SimpleMetricValue{Value: 1.0}, nil) + if failureMetric != "" { + EmitMetricToPrometheus(nil, nil, failureMetric, &SimpleMetricValue{Value: 1.0}, nil) + } return nil, fmt.Errorf("failed to fetch metrics from %s: %v", url, err) } defer func() { diff --git a/pkg/metrics/engine_fetcher_test.go b/pkg/metrics/engine_fetcher_test.go index fa2094b94..931248a61 100644 --- a/pkg/metrics/engine_fetcher_test.go +++ b/pkg/metrics/engine_fetcher_test.go @@ -25,6 +25,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -476,6 +477,76 @@ func TestEngineMetricsFetcher_RetryLogic(t *testing.T) { }) } +func TestEngineMetricsFetcher_FetchRawMetric(t *testing.T) { + optimizerMetrics := `# HELP vllm:deployment_replicas Number of suggested replicas. +# TYPE vllm:deployment_replicas gauge +vllm:deployment_replicas{model_name="deepseek-r1-distill-llama-8b"} 3 +` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/metrics/default/deepseek-r1-distill-llama-8b" { + w.WriteHeader(404) + return + } + w.WriteHeader(200) + fmt.Fprint(w, optimizerMetrics) + })) + defer server.Close() + + config := EngineMetricsFetcherConfig{ + Timeout: 5 * time.Second, + MaxRetries: 0, + BaseDelay: 10 * time.Millisecond, + MaxDelay: 100 * time.Millisecond, + InsecureTLS: true, + } + fetcher := NewEngineMetricsFetcherWithConfig(config) + ctx := context.Background() + + t.Run("FetchUnregisteredMetricFromExplicitPath", func(t *testing.T) { + url := server.URL + "/metrics/default/deepseek-r1-distill-llama-8b" + value, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:deployment_replicas") + + require.NoError(t, err) + assert.Equal(t, 3.0, value.GetSimpleValue()) + }) + + t.Run("MetricMissingFromResponse", func(t *testing.T) { + url := server.URL + "/metrics/default/deepseek-r1-distill-llama-8b" + _, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:missing_metric") + + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to fetch raw metric") + }) + + t.Run("WrongPathReturnsError", func(t *testing.T) { + url := server.URL + "/metrics" + _, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:deployment_replicas") + + require.Error(t, err) + }) + + t.Run("TransportFailureIsNotCountedAsEngineQueryFailure", func(t *testing.T) { + counter, cleanup := SetupCounterMetricsForTest(LLMEngineMetricsQueryFail, []string{"gateway_pod", "model"}) + defer cleanup() + + // A closed server yields a connection error, the only path that emits the counter. + deadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + deadURL := deadServer.URL + deadServer.Close() + + _, err := fetcher.FetchRawMetric(ctx, deadURL+"/metrics/default/model", "gpu-optimizer", "vllm:deployment_replicas") + require.Error(t, err) + assert.Equal(t, 0, testutil.CollectAndCount(counter), + "external metric fetch failures must not increment llm_engine_metrics_query_fail") + + // The engine path must keep counting, so the counter is still useful for engine scrapes. + _, err = fetcher.FetchTypedMetric(ctx, strings.TrimPrefix(deadURL, "http://"), "vllm", "test-pod", NumRequestsRunning) + require.Error(t, err) + assert.Equal(t, 1, testutil.CollectAndCount(counter), + "engine metric fetch failures should still increment llm_engine_metrics_query_fail") + }) +} + func TestEngineMetricsFetcher_BackoffDelay(t *testing.T) { config := DefaultEngineMetricsFetcherConfig() fetcher := NewEngineMetricsFetcherWithConfig(config) diff --git a/samples/autoscaling/external-metrics-kpa.yaml b/samples/autoscaling/external-metrics-kpa.yaml index fb6a1fb24..6529958d5 100644 --- a/samples/autoscaling/external-metrics-kpa.yaml +++ b/samples/autoscaling/external-metrics-kpa.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 3m + autoscaling.aibrix.ai/scale-down-cooldown-window: 3m spec: scalingStrategy: KPA minReplicas: 1 diff --git a/samples/autoscaling/kpa.yaml b/samples/autoscaling/kpa.yaml index 013043919..27f94b497 100644 --- a/samples/autoscaling/kpa.yaml +++ b/samples/autoscaling/kpa.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 3m + autoscaling.aibrix.ai/scale-down-cooldown-window: 3m spec: scalingStrategy: KPA minReplicas: 1 diff --git a/samples/autoscaling/optimizer-kpa.yaml b/samples/autoscaling/optimizer-kpa.yaml index 73692ab49..441ef2a9c 100644 --- a/samples/autoscaling/optimizer-kpa.yaml +++ b/samples/autoscaling/optimizer-kpa.yaml @@ -7,14 +7,14 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s spec: scalingStrategy: KPA minReplicas: 1 maxReplicas: 8 metricsSources: - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 - metricSourceType: domain + metricSourceType: external path: /metrics/default/deepseek-r1-distill-llama-8b protocolType: http targetMetric: vllm:deployment_replicas diff --git a/samples/deepseek-r1/deepseek-r1-autoscaling.yaml b/samples/deepseek-r1/deepseek-r1-autoscaling.yaml index cceda0edb..c5c6b6045 100644 --- a/samples/deepseek-r1/deepseek-r1-autoscaling.yaml +++ b/samples/deepseek-r1/deepseek-r1-autoscaling.yaml @@ -6,7 +6,7 @@ metadata: labels: app.kubernetes.io/name: aibrix annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 2m + autoscaling.aibrix.ai/scale-down-cooldown-window: 2m spec: scalingStrategy: KPA minReplicas: 1 diff --git a/samples/heterogeneous/deepseek-coder-7b-l20-podautoscaler.yaml b/samples/heterogeneous/deepseek-coder-7b-l20-podautoscaler.yaml index ab12c290a..56b03740a 100644 --- a/samples/heterogeneous/deepseek-coder-7b-l20-podautoscaler.yaml +++ b/samples/heterogeneous/deepseek-coder-7b-l20-podautoscaler.yaml @@ -5,14 +5,14 @@ metadata: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: aibrix annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s name: podautoscaler-deepseek-coder-7b-l20 namespace: default spec: maxReplicas: 10 metricsSources: - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 - metricSourceType: domain + metricSourceType: external path: /metrics/default/deepseek-coder-7b-l20 protocolType: http targetMetric: vllm:deployment_replicas diff --git a/samples/heterogeneous/deepseek-coder-7b-v100-podautoscaler.yaml b/samples/heterogeneous/deepseek-coder-7b-v100-podautoscaler.yaml index b71304de9..3320ccb65 100644 --- a/samples/heterogeneous/deepseek-coder-7b-v100-podautoscaler.yaml +++ b/samples/heterogeneous/deepseek-coder-7b-v100-podautoscaler.yaml @@ -5,14 +5,14 @@ metadata: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: aibrix annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 0s + autoscaling.aibrix.ai/scale-down-cooldown-window: 0s name: podautoscaler-deepseek-coder-7b-v100 namespace: default spec: maxReplicas: 10 metricsSources: - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080 - metricSourceType: domain + metricSourceType: external path: /metrics/default/deepseek-coder-7b-v100 protocolType: http targetMetric: vllm:deployment_replicas diff --git a/samples/volcano-engine/autoscaler.yaml b/samples/volcano-engine/autoscaler.yaml index 89e9cd322..3e351d13a 100644 --- a/samples/volcano-engine/autoscaler.yaml +++ b/samples/volcano-engine/autoscaler.yaml @@ -7,7 +7,7 @@ metadata: app.kubernetes.io/name: aibrix app.kubernetes.io/managed-by: kustomize annotations: - kpa.autoscaling.aibrix.ai/scale-down-delay: 5m + autoscaling.aibrix.ai/scale-down-cooldown-window: 5m spec: scalingStrategy: KPA minReplicas: 1