serve: greedy requests skip the full-vocabulary sampler chain (engine#560)

CMakeLists.txt

@@ -139,6 +139,7 @@ add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $
 add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $<TARGET_FILE:onebit>)
 add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $<TARGET_FILE:onebit>)
 add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $<TARGET_FILE:onebit>)
+add_test(NAME greedy_shortcut COMMAND ${CMAKE_SOURCE_DIR}/tests/greedy_shortcut.sh $<TARGET_FILE:onebit>)
 add_test(NAME device_route COMMAND ${CMAKE_SOURCE_DIR}/tests/device_route.sh $<TARGET_FILE:onebit>)
 # where --device npu finds the GGUF repack script: beside 1bit in a package (scripts/package-linux.sh),
 # else ONEBIT_Q4NX_REPACK; no NPU needed (the repack runs before the device)

app/serve.cpp

@@ -722,6 +722,13 @@ void forward(int port, std::shared_ptr<InFlight> held, int draft_max, const std:
         return;
     }
     if (drop_model) body.erase("model");
+    // Greedy short-circuit (issue #560): a request that asks for temperature 0 gets argmax by way of
+    // top_k 1 unless it names its own top_k. That skips the full-vocabulary top-p / min-p chain, which
+    // on a 262147-entry vocab costs about 2.1 ms per token (ZAYA1-8B: 118 -> 153 tok/s). A request that
+    // states any other temperature is untouched.
+    if (body.contains("temperature") && body["temperature"].is_number() &&
+        body["temperature"].get<double>() <= 0.0 && !body.contains("top_k"))
+        body["top_k"] = 1;
     // --precise: a follow-up turn differs from a cold run of the same conversation because its new tokens form a
     // different batch than the cold prefill does (different kernels, different reduction order), and a near-tie
     // then flips. The only way to make the two bitwise identical is to take the cold path: re-prefill the whole

docs/serve.md

@@ -526,6 +526,19 @@ with `-b 4096 -ub 4096`: an image is decoded as one ubatch, which models that at
 bidirectionally (ZAYA1-VL, Gemma 3) need. Zyphra's Zamba2-VL and ZAYA1-VL-8B were checked on the
 removed Vulkan build ([vulkan.md at 0baf286](https://github.com/1bit-MONSTER/engine/blob/0baf286/docs/vulkan.md)). The NPU and DwarfStar routes take no `--mmproj`.
 
+## Greedy requests (`temperature 0`)
+
+A request with `temperature: 0` (or any negative temperature) leaves nothing to sample: serve forwards
+it with `top_k: 1`, so llama-server takes the argmax and skips the top-p / min-p passes over the whole
+vocabulary. On a 262147-entry vocabulary those passes cost about **2.1 ms per token**, which is most of
+the decode step on a small model: ZAYA1-8B Q4_K_M goes from 118 to 153 tok/s at temperature 0, and
+Qwen3.8-27B gains in proportion to its own vocabulary (248320 entries).
+
+A request that names its own `top_k` keeps it — only the absent case is filled in — and a request with
+any other temperature is forwarded untouched, so sampling behaviour is unchanged everywhere it matters.
+`tests/greedy_shortcut.sh` (ctest `greedy_shortcut`) pins all four cases against `tests/fake_backend.py`
+without a GPU.
+
 ## Multi-token prediction (`--mtp`)
 
 `--mtp <head.gguf>` turns on llama-server's `draft-mtp` speculative decoding: the model's

tests/fake_backend.py

@@ -138,7 +138,10 @@ def do_POST(self):
             # (what the hybrid handoff sets; tests/hybrid_route.sh reads them back)
             self._json(200, {"id": "x", "model": "fake-backend-id", "choices": [dict(index=0, finish_reason="stop", **msg)],
                              "timings": timings, "fake_id_slot": req.get("id_slot", -1),
-                             "fake_cache_prompt": req.get("cache_prompt", None)})
+                             "fake_cache_prompt": req.get("cache_prompt", None),
+                             # the sampling fields serve forwards (tests/greedy_shortcut.sh)
+                             "fake_top_k": req.get("top_k", None),
+                             "fake_temperature": req.get("temperature", None)})
             # "fake_exit": N: the backend exits with status N shortly after this reply, on its own
             # (a crashed backend; serve must notice and not wait its SIGTERM grace for a dead pid)
             if "fake_exit" in req:

tests/greedy_shortcut.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# `1bit serve`'s greedy short-circuit (issue #560, docs/serve.md "Greedy requests"), without a GPU:
+# tests/fake_backend.py stands in for llama-server and echoes the sampling fields serve forwarded.
+#   - temperature 0 gets top_k 1 on both OpenAI routes (argmax without the full-vocab chain),
+#   - a request that names its own top_k keeps it,
+#   - a request with a real temperature is forwarded untouched,
+#   - a body with no temperature is not rewritten (llama-server's own default applies).
+#
+# usage: tests/greedy_shortcut.sh path/to/1bit
+set -uo pipefail
+
+bin=${1:?usage: greedy_shortcut.sh path/to/1bit}
+scratch=$(mktemp -d)
+trap 'kill -9 $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$scratch"' EXIT
+pid=
+fail=0
+check() { if eval "$2"; then echo "ok   $1"; else echo "FAIL $1"; fi; [ "$(eval "$2" >/dev/null 2>&1; echo $?)" = 0 ] || fail=1; }
+
+python3 - "$scratch" <<'PY'
+import struct, sys
+def gguf(path, arch="llama"):
+    kv = []
+    def s(x): b = x.encode(); return struct.pack("<Q", len(b)) + b
+    kv.append(s("general.architecture") + struct.pack("<I", 8) + s(arch))
+    open(path, "wb").write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(kv)) + b"".join(kv))
+gguf(sys.argv[1] + "/m.gguf")
+PY
+
+port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
+"$bin" serve -m "$scratch/m.gguf" --device auto --port "$port" \
+    --llama-server "$(dirname "$0")/fake_backend.py" >"$scratch/serve.log" 2>&1 &
+pid=$!
+for _ in $(seq 1 100); do curl -sf "localhost:$port/health" >/dev/null 2>&1 && break; sleep 0.2; done
+
+ask() {  # ask <path> <json> -> reply body
+    curl -s -m 20 -X POST "localhost:$port$1" -H 'Content-Type: application/json' -d "$2"
+}
+
+r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0}')
+check "temperature 0 on /v1/chat/completions gets top_k 1" '[[ "$r" == *"\"fake_top_k\": 1"* || "$r" == *"\"fake_top_k\":1"* ]]'
+
+r=$(ask /v1/completions '{"model":"m","prompt":"hi","max_tokens":4,"temperature":0}')
+check "temperature 0 on /v1/completions gets top_k 1" '[[ "$r" == *"\"fake_top_k\": 1"* || "$r" == *"\"fake_top_k\":1"* ]]'
+
+r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0,"top_k":5}')
+check "an explicit top_k is kept" '[[ "$r" == *"\"fake_top_k\": 5"* || "$r" == *"\"fake_top_k\":5"* ]]'
+
+r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0.7}')
+check "a real temperature is forwarded untouched (no top_k added)" '[[ "$r" == *"\"fake_top_k\": null"* || "$r" == *"\"fake_top_k\":null"* ]] && [[ "$r" == *"0.7"* ]]'
+
+r=$(ask /v1/completions '{"model":"m","prompt":"hi","max_tokens":4}')
+check "a body without temperature is not rewritten" '[[ "$r" == *"\"fake_top_k\": null"* || "$r" == *"\"fake_top_k\":null"* ]]'
+
+if [ $fail -ne 0 ]; then echo "--- $scratch/serve.log"; tail -20 "$scratch/serve.log"; echo FAIL; exit 1; fi
+echo PASS