diff --git a/CMakeLists.txt b/CMakeLists.txt index 8576f663..3b80fb96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,6 +139,7 @@ add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $ add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $) add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $) add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $) +add_test(NAME greedy_shortcut COMMAND ${CMAKE_SOURCE_DIR}/tests/greedy_shortcut.sh $) add_test(NAME device_route COMMAND ${CMAKE_SOURCE_DIR}/tests/device_route.sh $) # where --device npu finds the GGUF repack script: beside 1bit in a package (scripts/package-linux.sh), # else ONEBIT_Q4NX_REPACK; no NPU needed (the repack runs before the device) diff --git a/app/serve.cpp b/app/serve.cpp index 4eed99f3..e14b55d3 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -722,6 +722,13 @@ void forward(int port, std::shared_ptr held, int draft_max, const std: return; } if (drop_model) body.erase("model"); + // Greedy short-circuit (issue #560): a request that asks for temperature 0 gets argmax by way of + // top_k 1 unless it names its own top_k. That skips the full-vocabulary top-p / min-p chain, which + // on a 262147-entry vocab costs about 2.1 ms per token (ZAYA1-8B: 118 -> 153 tok/s). A request that + // states any other temperature is untouched. + if (body.contains("temperature") && body["temperature"].is_number() && + body["temperature"].get() <= 0.0 && !body.contains("top_k")) + body["top_k"] = 1; // --precise: a follow-up turn differs from a cold run of the same conversation because its new tokens form a // different batch than the cold prefill does (different kernels, different reduction order), and a near-tie // then flips. The only way to make the two bitwise identical is to take the cold path: re-prefill the whole diff --git a/docs/serve.md b/docs/serve.md index ca67558d..ba262c21 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -526,6 +526,19 @@ with `-b 4096 -ub 4096`: an image is decoded as one ubatch, which models that at bidirectionally (ZAYA1-VL, Gemma 3) need. Zyphra's Zamba2-VL and ZAYA1-VL-8B were checked on the removed Vulkan build ([vulkan.md at 0baf286](https://github.com/1bit-MONSTER/engine/blob/0baf286/docs/vulkan.md)). The NPU and DwarfStar routes take no `--mmproj`. +## Greedy requests (`temperature 0`) + +A request with `temperature: 0` (or any negative temperature) leaves nothing to sample: serve forwards +it with `top_k: 1`, so llama-server takes the argmax and skips the top-p / min-p passes over the whole +vocabulary. On a 262147-entry vocabulary those passes cost about **2.1 ms per token**, which is most of +the decode step on a small model: ZAYA1-8B Q4_K_M goes from 118 to 153 tok/s at temperature 0, and +Qwen3.8-27B gains in proportion to its own vocabulary (248320 entries). + +A request that names its own `top_k` keeps it — only the absent case is filled in — and a request with +any other temperature is forwarded untouched, so sampling behaviour is unchanged everywhere it matters. +`tests/greedy_shortcut.sh` (ctest `greedy_shortcut`) pins all four cases against `tests/fake_backend.py` +without a GPU. + ## Multi-token prediction (`--mtp`) `--mtp ` turns on llama-server's `draft-mtp` speculative decoding: the model's diff --git a/tests/fake_backend.py b/tests/fake_backend.py index f2b6f5cc..4ac21073 100755 --- a/tests/fake_backend.py +++ b/tests/fake_backend.py @@ -138,7 +138,10 @@ def do_POST(self): # (what the hybrid handoff sets; tests/hybrid_route.sh reads them back) self._json(200, {"id": "x", "model": "fake-backend-id", "choices": [dict(index=0, finish_reason="stop", **msg)], "timings": timings, "fake_id_slot": req.get("id_slot", -1), - "fake_cache_prompt": req.get("cache_prompt", None)}) + "fake_cache_prompt": req.get("cache_prompt", None), + # the sampling fields serve forwards (tests/greedy_shortcut.sh) + "fake_top_k": req.get("top_k", None), + "fake_temperature": req.get("temperature", None)}) # "fake_exit": N: the backend exits with status N shortly after this reply, on its own # (a crashed backend; serve must notice and not wait its SIGTERM grace for a dead pid) if "fake_exit" in req: diff --git a/tests/greedy_shortcut.sh b/tests/greedy_shortcut.sh new file mode 100755 index 00000000..2184f982 --- /dev/null +++ b/tests/greedy_shortcut.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# `1bit serve`'s greedy short-circuit (issue #560, docs/serve.md "Greedy requests"), without a GPU: +# tests/fake_backend.py stands in for llama-server and echoes the sampling fields serve forwarded. +# - temperature 0 gets top_k 1 on both OpenAI routes (argmax without the full-vocab chain), +# - a request that names its own top_k keeps it, +# - a request with a real temperature is forwarded untouched, +# - a body with no temperature is not rewritten (llama-server's own default applies). +# +# usage: tests/greedy_shortcut.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: greedy_shortcut.sh path/to/1bit} +scratch=$(mktemp -d) +trap 'kill -9 $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$scratch"' EXIT +pid= +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fi; [ "$(eval "$2" >/dev/null 2>&1; echo $?)" = 0 ] || fail=1; } + +python3 - "$scratch" <<'PY' +import struct, sys +def gguf(path, arch="llama"): + kv = [] + def s(x): b = x.encode(); return struct.pack(""$scratch/serve.log" 2>&1 & +pid=$! +for _ in $(seq 1 100); do curl -sf "localhost:$port/health" >/dev/null 2>&1 && break; sleep 0.2; done + +ask() { # ask -> reply body + curl -s -m 20 -X POST "localhost:$port$1" -H 'Content-Type: application/json' -d "$2" +} + +r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0}') +check "temperature 0 on /v1/chat/completions gets top_k 1" '[[ "$r" == *"\"fake_top_k\": 1"* || "$r" == *"\"fake_top_k\":1"* ]]' + +r=$(ask /v1/completions '{"model":"m","prompt":"hi","max_tokens":4,"temperature":0}') +check "temperature 0 on /v1/completions gets top_k 1" '[[ "$r" == *"\"fake_top_k\": 1"* || "$r" == *"\"fake_top_k\":1"* ]]' + +r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0,"top_k":5}') +check "an explicit top_k is kept" '[[ "$r" == *"\"fake_top_k\": 5"* || "$r" == *"\"fake_top_k\":5"* ]]' + +r=$(ask /v1/chat/completions '{"model":"m","messages":[{"role":"user","content":"hi"}],"max_tokens":4,"temperature":0.7}') +check "a real temperature is forwarded untouched (no top_k added)" '[[ "$r" == *"\"fake_top_k\": null"* || "$r" == *"\"fake_top_k\":null"* ]] && [[ "$r" == *"0.7"* ]]' + +r=$(ask /v1/completions '{"model":"m","prompt":"hi","max_tokens":4}') +check "a body without temperature is not rewritten" '[[ "$r" == *"\"fake_top_k\": null"* || "$r" == *"\"fake_top_k\":null"* ]]' + +if [ $fail -ne 0 ]; then echo "--- $scratch/serve.log"; tail -20 "$scratch/serve.log"; echo FAIL; exit 1; fi +echo PASS