[Bug] Fix external metric fetching for GPU optimizer autoscaling and replace dead scale-down annotation in samples

benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/kpa.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 3m
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 3m
 spec:
   scalingStrategy: KPA
   minReplicas: 1

benchmarks/scenarios/autoscaling/deepseek-llm-7b-chat/optimizer-kpa.yaml

@@ -7,14 +7,14 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 0s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 0s
 spec:
   scalingStrategy: KPA 
   minReplicas: 1
   maxReplicas: 8
   metricsSources:
     - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
-      metricSourceType: domain
+      metricSourceType: external
       path: /metrics/default/deepseek-llm-7b-chat
       protocolType: http
       targetMetric: vllm:deployment_replicas

config/samples/autoscaling_v1alpha1_mock_llama.yaml

@@ -8,7 +8,7 @@ metadata:
   annotations:
     autoscaling.aibrix.ai/max-scale-up-rate: "2"
     autoscaling.aibrix.ai/max-scale-down-rate: "2"
-    kpa.autoscaling.aibrix.ai/scale-down-delay: "60s"
+    autoscaling.aibrix.ai/scale-down-cooldown-window: "60s"
   namespace: aibrix-system
 spec:
   scaleTargetRef:

development/app/config/heterogeneous/simulator_a40/patch_podautoscaler_a40.yaml

@@ -4,14 +4,14 @@ kind: PodAutoscaler
 metadata:
   name: podautoscaler-simulator-llama2-7b-a40
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 0s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 0s
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
     name: simulator-llama2-7b-a40
   metricsSources: 
-    - metricSourceType: domain
+    - metricSourceType: external
       protocolType: http
       endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
       path: /metrics/default/simulator-llama2-7b-a40

development/app/config/simulator/patch_podautoscaler_a100.yaml

@@ -4,14 +4,14 @@ kind: PodAutoscaler
 metadata:
   name: podautoscaler-simulator-llama2-7b-a100
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 0s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 0s
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
     name: simulator-llama2-7b-a100
   metricsSources: 
-    - metricSourceType: domain
+    - metricSourceType: external
       protocolType: http
       endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
       path: /metrics/default/simulator-llama2-7b-a100

development/app/config/templates/podautoscaler/podautoscaler_kpa.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 30s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 30s
   namespace: default
 spec:
   scaleTargetRef:

development/tutorials/distributed/fleet-autoscaling.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     app.kubernetes.io/name: aibrix
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 1m
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 1m
 spec:
   scalingStrategy: KPA
   minReplicas: 1

pkg/controller/podautoscaler/metrics/fetcher.go

@@ -19,6 +19,7 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -272,9 +273,16 @@ func (f *ExternalMetricsFetcher) fetchFromGPUOptimizer(ctx context.Context, pod
 		"path", source.Path,
 		"metric", source.TargetMetric)
 
-	// Use the centralized engine fetcher for external HTTP calls
-	// This gives us a global value that we need to adapt to per-pod semantics
-	metricValue, err := f.engineFetcher.FetchTypedMetric(ctx, source.Endpoint, "external", "gpu-optimizer", source.TargetMetric)
+	protocol := source.ProtocolType
+	if protocol == "" {
+		protocol = autoscalingv1alpha1.HTTP
+	}
+	url := fmt.Sprintf("%s://%s/%s", protocol, source.Endpoint, strings.TrimLeft(source.Path, "/"))
+
+	// External metrics are not engine metrics: fetch the raw metric directly from the
+	// configured endpoint and path instead of resolving through the central registry.
+	// This gives us a global value that we need to adapt to per-pod semantics.
+	metricValue, err := f.engineFetcher.FetchRawMetric(ctx, url, source.Endpoint, source.TargetMetric)
 	if err != nil {
 		klog.Warningf("Failed to fetch metric %s from GPU-Optimizer %s: %v",
 			source.TargetMetric, source.Endpoint, err)

pkg/metrics/engine_fetcher.go

@@ -124,10 +124,8 @@ func (ef *EngineMetricsFetcher) FetchTypedMetric(ctx context.Context, endpoint,
 			klog.V(4).InfoS("Retrying typed metric fetch from engine endpoint",
 				"attempt", attempt, "delay", delay, "identifier", identifier, "metric", metricName)
 
-			select {
-			case <-ctx.Done():
-				return nil, ctx.Err()
-			case <-time.After(delay):
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return nil, err
 			}
 		}
 
@@ -156,6 +154,48 @@ func (ef *EngineMetricsFetcher) FetchTypedMetric(ctx context.Context, endpoint,
 		metricName, identifier, ef.config.MaxRetries+1)
 }
 
+// FetchRawMetric fetches a metric by its raw Prometheus name from an explicit metrics URL,
+// bypassing the central metric registry. External sources such as the GPU optimizer expose
+// caller-defined metrics on caller-defined paths, so neither the registry's metric
+// definitions nor its per-engine paths apply to them.
+func (ef *EngineMetricsFetcher) FetchRawMetric(ctx context.Context, url, identifier, rawMetricName string) (MetricValue, error) {
+	for attempt := 0; attempt <= ef.config.MaxRetries; attempt++ {
+		if attempt > 0 {
+			delay := ef.calculateBackoffDelay(attempt)
+			klog.V(4).InfoS("Retrying raw metric fetch",
+				"attempt", attempt, "delay", delay, "identifier", identifier, "metric", rawMetricName)
+
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return nil, err
+			}
+		}
+
+		// Do not attribute failures of external sources to the engine failure counter.
+		allMetrics, err := ef.fetchMetricsFromURL(ctx, url, "")
+		if err != nil {
+			klog.V(4).InfoS("Failed to fetch metrics from URL",
+				"attempt", attempt+1, "identifier", identifier, "url", url, "error", err)
+			continue
+		}
+
+		family, exists := allMetrics[rawMetricName]
+		if !exists || len(family.Metric) == 0 {
+			klog.V(4).InfoS("Raw metric not found in response",
+				"attempt", attempt+1, "identifier", identifier, "metric", rawMetricName)
+			continue
+		}
+
+		metricValue, err := GetCounterGaugeValue(family.Metric[0], family.GetType())
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse raw metric %s from %s: %w", rawMetricName, identifier, err)
+		}
+		return metricValue, nil
+	}
+
+	return nil, fmt.Errorf("failed to fetch raw metric %s from %s after %d attempts",
+		rawMetricName, identifier, ef.config.MaxRetries+1)
+}
+
 // FetchAllTypedMetrics fetches all available typed metrics from an engine endpoint
 func (ef *EngineMetricsFetcher) FetchAllTypedMetrics(ctx context.Context, endpoint, engineType, identifier string, requestedMetrics []string) (*EngineMetricsResult, error) {
 	result := &EngineMetricsResult{
@@ -179,10 +219,8 @@ func (ef *EngineMetricsFetcher) FetchAllTypedMetrics(ctx context.Context, endpoi
 			klog.V(4).InfoS("Retrying all typed metrics fetch from engine endpoint",
 				"attempt", attempt, "delay", delay, "identifier", identifier)
 
-			select {
-			case <-ctx.Done():
-				return nil, ctx.Err()
-			case <-time.After(delay):
+			if err := sleepWithContext(ctx, delay); err != nil {
+				return nil, err
 			}
 		}
 
@@ -276,6 +314,21 @@ func (ef *EngineMetricsFetcher) calculateBackoffDelay(attempt int) time.Duration
 	return delay
 }
 
+// sleepWithContext blocks for the given delay or until ctx is done, whichever comes first.
+// It uses an explicit timer that is stopped on early return so a cancelled context does not
+// leave a pending timer behind, unlike time.After, whose timer cannot be stopped.
+func sleepWithContext(ctx context.Context, delay time.Duration) error {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
 // getAvailableMetricsForEngine returns all metrics available for a given engine type
 func (ef *EngineMetricsFetcher) getAvailableMetricsForEngine(engineType string) []string {
 	var availableMetrics []string
@@ -423,8 +476,16 @@ func (ef *EngineMetricsFetcher) parseMetricInstance(familyMetric *dto.Metric, me
 	return nil, fmt.Errorf("unsupported metric type for raw parsing: %v", metric.MetricType)
 }
 
-// fetchAllMetricsFromURL performs a single HTTP request and parses all Prometheus metrics
+// fetchAllMetricsFromURL performs a single HTTP request against an engine endpoint and parses
+// all Prometheus metrics. Transport failures are counted in llm_engine_metrics_query_fail.
 func (ef *EngineMetricsFetcher) fetchAllMetricsFromURL(ctx context.Context, url string) (map[string]*dto.MetricFamily, error) {
+	return ef.fetchMetricsFromURL(ctx, url, LLMEngineMetricsQueryFail)
+}
+
+// fetchMetricsFromURL performs a single HTTP request and parses all Prometheus metrics in the
+// response. If failureMetric is non-empty, transport failures increment that counter; callers
+// scraping non-engine sources pass an empty name so their failures are not attributed to engines.
+func (ef *EngineMetricsFetcher) fetchMetricsFromURL(ctx context.Context, url, failureMetric string) (map[string]*dto.MetricFamily, error) {
 	// Use our configured HTTP client with the existing parsing logic
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
@@ -433,7 +494,9 @@ func (ef *EngineMetricsFetcher) fetchAllMetricsFromURL(ctx context.Context, url
 
 	resp, err := ef.client.Do(req)
 	if err != nil {
-		EmitMetricToPrometheus(nil, nil, LLMEngineMetricsQueryFail, &SimpleMetricValue{Value: 1.0}, nil)
+		if failureMetric != "" {
+			EmitMetricToPrometheus(nil, nil, failureMetric, &SimpleMetricValue{Value: 1.0}, nil)
+		}
 		return nil, fmt.Errorf("failed to fetch metrics from %s: %v", url, err)
 	}
 	defer func() {

pkg/metrics/engine_fetcher_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -476,6 +477,76 @@ func TestEngineMetricsFetcher_RetryLogic(t *testing.T) {
 	})
 }
 
+func TestEngineMetricsFetcher_FetchRawMetric(t *testing.T) {
+	optimizerMetrics := `# HELP vllm:deployment_replicas Number of suggested replicas.
+# TYPE vllm:deployment_replicas gauge
+vllm:deployment_replicas{model_name="deepseek-r1-distill-llama-8b"} 3
+`
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/metrics/default/deepseek-r1-distill-llama-8b" {
+			w.WriteHeader(404)
+			return
+		}
+		w.WriteHeader(200)
+		fmt.Fprint(w, optimizerMetrics)
+	}))
+	defer server.Close()
+
+	config := EngineMetricsFetcherConfig{
+		Timeout:     5 * time.Second,
+		MaxRetries:  0,
+		BaseDelay:   10 * time.Millisecond,
+		MaxDelay:    100 * time.Millisecond,
+		InsecureTLS: true,
+	}
+	fetcher := NewEngineMetricsFetcherWithConfig(config)
+	ctx := context.Background()
+
+	t.Run("FetchUnregisteredMetricFromExplicitPath", func(t *testing.T) {
+		url := server.URL + "/metrics/default/deepseek-r1-distill-llama-8b"
+		value, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:deployment_replicas")
+
+		require.NoError(t, err)
+		assert.Equal(t, 3.0, value.GetSimpleValue())
+	})
+
+	t.Run("MetricMissingFromResponse", func(t *testing.T) {
+		url := server.URL + "/metrics/default/deepseek-r1-distill-llama-8b"
+		_, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:missing_metric")
+
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "failed to fetch raw metric")
+	})
+
+	t.Run("WrongPathReturnsError", func(t *testing.T) {
+		url := server.URL + "/metrics"
+		_, err := fetcher.FetchRawMetric(ctx, url, "gpu-optimizer", "vllm:deployment_replicas")
+
+		require.Error(t, err)
+	})
+
+	t.Run("TransportFailureIsNotCountedAsEngineQueryFailure", func(t *testing.T) {
+		counter, cleanup := SetupCounterMetricsForTest(LLMEngineMetricsQueryFail, []string{"gateway_pod", "model"})
+		defer cleanup()
+
+		// A closed server yields a connection error, the only path that emits the counter.
+		deadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+		deadURL := deadServer.URL
+		deadServer.Close()
+
+		_, err := fetcher.FetchRawMetric(ctx, deadURL+"/metrics/default/model", "gpu-optimizer", "vllm:deployment_replicas")
+		require.Error(t, err)
+		assert.Equal(t, 0, testutil.CollectAndCount(counter),
+			"external metric fetch failures must not increment llm_engine_metrics_query_fail")
+
+		// The engine path must keep counting, so the counter is still useful for engine scrapes.
+		_, err = fetcher.FetchTypedMetric(ctx, strings.TrimPrefix(deadURL, "http://"), "vllm", "test-pod", NumRequestsRunning)
+		require.Error(t, err)
+		assert.Equal(t, 1, testutil.CollectAndCount(counter),
+			"engine metric fetch failures should still increment llm_engine_metrics_query_fail")
+	})
+}
+
 func TestEngineMetricsFetcher_BackoffDelay(t *testing.T) {
 	config := DefaultEngineMetricsFetcherConfig()
 	fetcher := NewEngineMetricsFetcherWithConfig(config)

samples/autoscaling/external-metrics-kpa.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 3m
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 3m
 spec:
   scalingStrategy: KPA
   minReplicas: 1

samples/autoscaling/kpa.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 3m
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 3m
 spec:
   scalingStrategy: KPA
   minReplicas: 1

samples/autoscaling/optimizer-kpa.yaml

@@ -7,14 +7,14 @@ metadata:
     app.kubernetes.io/name: aibrix
     app.kubernetes.io/managed-by: kustomize
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 0s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 0s
 spec:
   scalingStrategy: KPA 
   minReplicas: 1
   maxReplicas: 8
   metricsSources:
   - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
-    metricSourceType: domain
+    metricSourceType: external
     path: /metrics/default/deepseek-r1-distill-llama-8b
     protocolType: http
     targetMetric: vllm:deployment_replicas

samples/deepseek-r1/deepseek-r1-autoscaling.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     app.kubernetes.io/name: aibrix
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 2m
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 2m
 spec:
   scalingStrategy: KPA
   minReplicas: 1

samples/heterogeneous/deepseek-coder-7b-l20-podautoscaler.yaml

@@ -5,14 +5,14 @@ metadata:
     app.kubernetes.io/managed-by: kustomize
     app.kubernetes.io/name: aibrix
   annotations:
-    kpa.autoscaling.aibrix.ai/scale-down-delay: 0s
+    autoscaling.aibrix.ai/scale-down-cooldown-window: 0s
   name: podautoscaler-deepseek-coder-7b-l20
   namespace: default
 spec:
   maxReplicas: 10
   metricsSources:
   - endpoint: aibrix-gpu-optimizer.aibrix-system.svc.cluster.local:8080
-    metricSourceType: domain
+    metricSourceType: external
     path: /metrics/default/deepseek-coder-7b-l20
     protocolType: http
     targetMetric: vllm:deployment_replicas