From 41f3ae4129ee08d07555f7d25f18156b121da0de Mon Sep 17 00:00:00 2001 From: 100-JM Date: Sun, 4 Oct 2026 03:11:40 +0900 Subject: [PATCH] Fix avg_time_queue_us under-reporting in k8s-onprem autoscaling rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prometheus-adapter rule averaged per-model queue-time ratios, so idle models diluted the pod's reported queue time, and the 1+delta denominator divided by n+1 requests instead of n. Sum both per-model counters over the pod before dividing, guard the zero-request case with clamp_min, and use increase instead of delta since both series are counters that reset on server restart. Verified with promtool: one pod, two models, traffic to one (100 req/30s, 1000us queued each) — the old rule reports 495.05, the new rule 1000. Fixes #8979 Co-Authored-By: Claude Fable 5 --- deploy/k8s-onprem/values.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index fdbb0cc714..6804ec745f 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -81,4 +81,9 @@ prometheus-adapter: name: matches: "nv_inference_queue_duration_us" as: "avg_time_queue_us" - metricsQuery: 'avg(delta(nv_inference_queue_duration_us{<<.LabelMatchers>>}[30s])/(1+delta(nv_inference_request_success{<<.LabelMatchers>>}[30s]))) by (<<.GroupBy>>)' + # Mean time a request spent queued on the pod across all models it serves. + # Both counters are per-model series, so they must be summed over the pod + # before dividing; averaging per-model ratios lets idle models dilute the + # value. clamp_min guards the division by zero without shifting the result, + # and increase (not delta) handles counter resets on server restart. + metricsQuery: 'sum(increase(nv_inference_queue_duration_us{<<.LabelMatchers>>}[30s])) by (<<.GroupBy>>) / clamp_min(sum(increase(nv_inference_request_success{<<.LabelMatchers>>}[30s])) by (<<.GroupBy>>), 1)'