Server: 1bit-server speaks Lemonade's backend protocol

CMakeLists.txt

@@ -37,7 +37,8 @@ add_library(onebit_core STATIC
     engine/core/model_config.cpp
     engine/core/parallel.cpp
     engine/core/unicode.cpp
-    engine/core/tokenizer.cpp)
+    engine/core/tokenizer.cpp
+    engine/core/sampler.cpp)
 target_include_directories(onebit_core PUBLIC engine/core)
 find_package(Threads REQUIRED)
 target_link_libraries(onebit_core PUBLIC Threads::Threads)
@@ -46,11 +47,25 @@ add_library(onebit_cpu STATIC engine/backends/cpu/cpu_model.cpp)
 target_include_directories(onebit_cpu PUBLIC engine/backends/cpu)
 target_link_libraries(onebit_cpu PUBLIC onebit_core)
 
+include(cmake/deps.cmake)
+
+add_library(onebit_server_lib STATIC
+    engine/server/text_stream.cpp
+    engine/server/chat_template.cpp
+    engine/server/generator.cpp
+    engine/server/openai.cpp)
+target_include_directories(onebit_server_lib PUBLIC engine/server)
+target_link_libraries(onebit_server_lib PUBLIC onebit_cpu onebit_minja)
+
+add_executable(onebit_server engine/server/main.cpp)
+set_target_properties(onebit_server PROPERTIES OUTPUT_NAME 1bit-server)
+target_link_libraries(onebit_server PRIVATE onebit_server_lib onebit_httplib)
+
 # Unit tests: synthetic data only, so they run anywhere (including CI).
-foreach(t test_gguf test_dequant test_unicode)
+foreach(t test_gguf test_dequant test_unicode test_sampler test_text_stream test_openai)
     add_executable(${t} tests/${t}.cpp)
     target_include_directories(${t} PRIVATE tests)
-    target_link_libraries(${t} PRIVATE onebit_core)
+    target_link_libraries(${t} PRIVATE onebit_server_lib)
     add_test(NAME ${t} COMMAND ${t})
 endforeach()
 
@@ -64,13 +79,23 @@ set(ENGINE_GOLDEN_MODEL "" CACHE FILEPATH "GGUF model for the golden test")
 set(ENGINE_GOLDEN_DIR "" CACHE PATH "Golden logits directory for the golden test")
 if(ENGINE_GOLDEN_MODEL AND ENGINE_GOLDEN_DIR)
     add_test(NAME golden_cpu COMMAND golden_cpu ${ENGINE_GOLDEN_MODEL} ${ENGINE_GOLDEN_DIR})
+    # The server end to end over HTTP, with Lemonade's launch flags.
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    add_test(NAME server_e2e
+             COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/server_e2e.py --server $<TARGET_FILE:onebit_server>
+                     --model ${ENGINE_GOLDEN_MODEL} --golden ${ENGINE_GOLDEN_DIR})
 endif()
 
 # Tokenizer golden test: encode/decode/pre-token split against HF tokenizers.
 # Runs everywhere: the vocab-only GGUF and the cases are committed
 # (tools/golden/make_tokenizer_golden.py, docs/tokenizer.md).
 add_executable(golden_tokenizer tests/golden_tokenizer.cpp)
 target_link_libraries(golden_tokenizer PRIVATE onebit_core)
+add_executable(golden_template tests/golden_template.cpp)
+target_link_libraries(golden_template PRIVATE onebit_server_lib)
+add_test(NAME golden_template_qwen3
+         COMMAND golden_template ${CMAKE_SOURCE_DIR}/tests/golden/qwen3-tokenizer/vocab.gguf
+                 ${CMAKE_SOURCE_DIR}/tests/golden/qwen3-template/cases.jsonl)
 add_test(NAME golden_tokenizer_qwen3
          COMMAND golden_tokenizer ${CMAKE_SOURCE_DIR}/tests/golden/qwen3-tokenizer/vocab.gguf
                  ${CMAKE_SOURCE_DIR}/tests/golden/qwen3-tokenizer/cases.tsv)

NOTICE

@@ -6,3 +6,8 @@ This product is licensed under the Apache License, Version 2.0 (see LICENSE).
 engine/core/unicode_tables.inc is generated from the Unicode Character
 Database (via Python's unicodedata module), Copyright Unicode, Inc., used
 under the Unicode License v3 (https://www.unicode.org/license.txt).
+
+Third-party components fetched at build time (cmake/deps.cmake), all MIT:
+  nlohmann/json 3.12.0       Copyright (c) 2013-2025 Niels Lohmann
+  cpp-httplib 0.57.1         Copyright (c) 2017 yhirose
+  google/minja (021c2293)    Copyright 2024 Google LLC

README.md

@@ -21,10 +21,12 @@ XDNA 2 NPU, Radeon iGPU (Vulkan and HRX), and a CPU reference path. It serves th
 llama-server-compatible OpenAI HTTP API, so [Lemonade](https://github.com/lemonade-sdk/lemonade)
 and any OpenAI client can drive it as a drop-in backend.
 
-> **Status: scaffold.** Nothing runs yet. Components land one at a time, each with a
-> test against the CPU reference (see [CONTRIBUTING.md](CONTRIBUTING.md)). The
-> development history lives in [1bit-MONSTER](https://github.com/1bit-MONSTER/1bit-MONSTER);
-> this repository contains only code that has been verified.
+> **Status:** the CPU reference, tokenizer and server work end to end on Qwen3
+> (`1bit-server`, see [docs/server.md](docs/server.md)). The NPU and GPU backends
+> come next. Components land one at a time, each with a test against the CPU
+> reference (see [CONTRIBUTING.md](CONTRIBUTING.md)). The development history
+> lives in [1bit-MONSTER](https://github.com/1bit-MONSTER/1bit-MONSTER); this
+> repository contains only code that has been verified.
 
 ## Layout
 
@@ -35,7 +37,7 @@ and any OpenAI client can drive it as a drop-in backend.
 | `engine/backends/npu/` | XDNA 2 via XRT, full-ELF kernels only (no xclbins) |
 | `engine/backends/gpu/` | llama.cpp (AMD-Ecosystem fork) with the Vulkan and HRX20 devices |
 | `engine/backends/cpu/` | fp32 reference forward pass; the correctness oracle |
-| `engine/server/` | llama-server-compatible HTTP: `/v1/chat/completions`, `/v1/completions`, `/v1/models`, `/health` |
+| `engine/server/` | `1bit-server`: Lemonade's backend protocol, i.e. `/health`, `/v1/models`, `/v1/chat/completions`, `/v1/completions` |
 | `tests/` | Golden-logit tests per backend against the CPU reference |
 | `lemonade/` | Upstream patch: `BackendDescriptor` + `WrappedServer` for this engine |
 

cmake/deps.cmake

@@ -0,0 +1,43 @@
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Third-party dependencies, each pinned to a release (or commit) and its
+# sha256. All are header-only and MIT-licensed; see NOTICE.
+include(FetchContent)
+
+FetchContent_Declare(nlohmann_json
+    URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
+    URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa)
+set(JSON_BuildTests OFF CACHE INTERNAL "")
+set(JSON_Install OFF CACHE INTERNAL "")
+
+# SOURCE_SUBDIR points at a directory that does not exist, so FetchContent
+# only downloads these and does not run their own CMake projects.
+FetchContent_Declare(cpp_httplib
+    URL https://github.com/yhirose/cpp-httplib/archive/refs/tags/v0.57.1.tar.gz
+    URL_HASH SHA256=5c9e56b6638eb415dc451ac57531e84931fcb50e261ca3dd74eec6b484fb4712
+    SOURCE_SUBDIR _none)
+FetchContent_Declare(minja
+    URL https://github.com/google/minja/archive/021c2293c187789ef13d56c6cfd89c9b134fd80f.tar.gz
+    URL_HASH SHA256=dc3ddd37497b79a4cd35fd41550e22b3b0139439de8be6eab3d2d024fed47bb4
+    SOURCE_SUBDIR _none)
+
+FetchContent_MakeAvailable(nlohmann_json cpp_httplib minja)
+
+add_library(onebit_httplib INTERFACE)
+target_include_directories(onebit_httplib SYSTEM INTERFACE ${cpp_httplib_SOURCE_DIR})
+add_library(onebit_minja INTERFACE)
+target_include_directories(onebit_minja SYSTEM INTERFACE ${minja_SOURCE_DIR}/include)
+target_link_libraries(onebit_minja INTERFACE nlohmann_json::nlohmann_json)

docs/PORTING.md

@@ -28,7 +28,7 @@ and what has to be true before it lands here. Refs are branches or commits in th
 | NPU 16-tile layer kernel | branch `bench/fastlane-16tile-corrections-2026-09-22` (435bf36e7) | 24/24 tokens, summed KL 0.000466, 0.340 ms/layer | rebuild from source, reproduce |
 | GPU backend | AMD-Ecosystem/llama.cpp fork, `GGML_VULKAN` + `GGML_HRX2`; recipe on `fix/zaya-lmhead-evidence` | Vulkan0 74.8 tok/s, HRX20 18.4 on zaya1-8b; Q4NX is HRX20-only | pinned submodule, linked (no dlopen of copied structs) |
 | Router | `src/model_router.cpp` (Q4NX rule) + Laya scorer, branch `backup/laya-and-results-2026-09-22` | Laya not yet checked against its Python reference | Laya matches its Python reference within a stated tolerance |
-| Server | `src/server/` | works | llama-server flag and endpoint compatibility test |
+| Server | `src/server/` | **landed** as a new implementation (docs/server.md) | Lemonade's flags and endpoints, end-to-end test |
 | Lemonade recipe | lemonade `src/cpp/include/lemon/backends/*` pattern | n/a | loads Qwen3-0.6B through a local Lemonade build |
 
 Known traps carried over:

docs/server.md

@@ -0,0 +1,128 @@
+<!--
+Copyright 2026 bong-water-water-bong
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Server
+
+`1bit-server` is the engine's HTTP front end. It speaks the backend protocol
+Lemonade uses for its GGUF recipes (`llamacpp`, `llamacpp-hrx`): the launch
+flags, a `/health` probe, and the OpenAI endpoints with llama-server's
+`timings` extension. Lemonade can run it wherever it runs those backends today.
+
+Only the protocol is shared with llama-server. The inference is this repository's own:
+
+- model code (`engine/backends`)
+- tokenizer, sampler and chat-template rendering (`engine/core`, `engine/server`)
+
+## Launch
+
+```bash
+1bit-server -m model.gguf --ctx-size 4096 --device cpu --port 8080 --jinja --metrics --parallel 1
+```
+
+| Flag | Meaning |
+|---|---|
+| `-m, --model` | GGUF model (required) |
+| `-c, --ctx-size` | context length (default: min(training context, 4096)) |
+| `--host`, `--port` | listen address (default `127.0.0.1:8080`) |
+| `--device` | `auto` or `cpu`. More devices are added with each backend; an unknown device is an error, never a silent fallback |
+| `-t, --threads` | CPU threads |
+| `-a, --alias` | model id reported by the API |
+| `-np, --parallel` | only `1` |
+| `--jinja`, `--metrics` | accepted for compatibility. The GGUF chat template is always used; there is no `/metrics` yet |
+
+Unknown flags are rejected, so a misconfigured launch fails loudly.
+
+## Endpoints
+
+| Endpoint | Notes |
+|---|---|
+| `GET /health` | `503 {"error":{"message":"Loading model"}}` while loading, then `200 {"status":"ok"}` |
+| `GET /v1/models` | one model, id = alias |
+| `POST /v1/chat/completions` | messages rendered with the GGUF's Jinja template; streaming or not |
+| `POST /v1/completions` | raw prompt; streaming or not |
+
+Every response carries OpenAI `usage` (including
+`prompt_tokens_details.cached_tokens`) and llama-server `timings` (`prompt_n`,
+`prompt_ms`, `predicted_n`, `predicted_per_second`, `cache_n` and others). These
+are the fields Lemonade's telemetry reads. Streams end with `data: [DONE]`. The
+final chunk carries `finish_reason` and `timings`, and
+`stream_options.include_usage` adds a usage chunk.
+
+### Request fields
+
+- **Sampling:** `temperature` (0 = greedy), `top_p`, `top_k`, `min_p` and `seed`.
+  The defaults match llama-server: 0.8, 0.95, 40 and 0.05, with a random seed.
+- **Limits and streaming:** `max_completion_tokens` / `max_tokens` / `n_predict`,
+  `stop` (a string or an array), `stream` and `stream_options`.
+- **Chat:** `chat_template_kwargs` (for example `{"enable_thinking": false}`),
+  `reasoning_format` (`none` keeps `<think>` in `content`), and `tools`
+  (rendered into the prompt).
+- **Other fields** are ignored.
+- **Rejected with 400**, because they would otherwise be silently wrong:
+  `n > 1`, non-text content parts, batched prompts, and a prompt that does not
+  fit the context (`exceed_context_size_error`).
+
+### Reasoning
+
+A leading `<think>...</think>` block goes to `message.reasoning_content` (in
+streams, `delta.reasoning_content`), and the answer goes to `content`. This is
+the split Lemonade's chat, MCP and Ollama paths read. When the template itself
+opens the block (the prompt ends in `<think>`), the output starts in reasoning.
+
+### Prompt cache
+
+The previous request's tokens stay in the KV cache. A new request reuses the
+longest shared prefix and evaluates only the rest, reported as `cache_n`. That
+makes multi-turn chat cost one turn, not the whole conversation.
+
+## Verification
+
+`tests/server_e2e.py` launches the binary with Lemonade's flags and checks the
+contract over HTTP. It runs in `ctest` when a golden model is configured. On
+Qwen3-0.6B it checks:
+
+- **Correctness:** the greedy `/v1/completions` text is byte-identical to HF
+  transformers' fp32 greedy continuation from the golden, and `prompt_tokens`
+  matches HF's tokenization.
+- **Streaming:**
+  - streamed text equals non-streamed text, for completions and for chat
+    (both `content` and `reasoning_content`)
+  - there is exactly one final chunk, with `finish_reason` and `timings`
+  - the usage chunk appears
+  - the stream ends with `[DONE]`
+- **Prefix cache:** a repeated request reuses the whole prompt but its last
+  token (`cache_n = n_prompt - 1`) and gives the same text.
+- **Stops and chat:**
+  - stop strings truncate before the match, with `finish_reason: stop`
+  - chat stops on the end-of-turn token
+  - `enable_thinking: false` gives plain content
+  - thinking on splits the reasoning out
+- **Errors:** bad JSON, `n = 2` and an over-long prompt give 400; an unknown route gives 404.
+
+Unit tests (CI): `test_text_stream` covers UTF-8 split across tokens, stop
+strings across tokens and reasoning routing byte by byte. `test_openai` covers
+request parsing and response shapes, checked structurally. `test_sampler`
+covers greedy, seeding, top-k, top-p and min-p. `golden_template_qwen3`
+renders 12 conversations byte-identical to HF `apply_chat_template`, including
+tools, tool calls and the thinking toggle.
+
+## Not yet
+
+- **Tool-call output parsing.** Tools are rendered into the prompt, but a
+  generated `<tool_call>` block is returned as text, not as `tool_calls`.
+- **Endpoints:** `/v1/responses`, `/v1/embeddings`, `/tokenize`, `/slots` and `/metrics`.
+- **Concurrency:** more than one concurrent sequence. Requests queue.
+- **Devices:** anything but the CPU reference. The NPU and GPU backends plug in behind the same server.

engine/backends/cpu/cpu_model.cpp

@@ -26,9 +26,8 @@ namespace onebit {
 
 namespace {
 
-// KV cache length. Kept well under the training context so the reference
-// stays small; raise it when a test needs longer sequences.
-constexpr uint32_t kMaxCtx = 4096;
+// Default KV cache length when the caller does not choose one.
+constexpr uint32_t kDefaultCtx = 4096;
 
 void rms_norm(const float* x, const float* w, float* out, uint32_t n, float eps) {
     double ss = 0.0;
@@ -87,7 +86,8 @@ class Loader {
 
 }  // namespace
 
-std::expected<CpuModel, std::string> CpuModel::load(const std::string& gguf_path, size_t n_threads) {
+std::expected<CpuModel, std::string> CpuModel::load(const std::string& gguf_path, size_t n_threads,
+                                                    uint32_t n_ctx) {
     auto file = GgufFile::open(gguf_path);
     if (!file) return std::unexpected(file.error());
     auto cfg = read_model_config(*file);
@@ -130,7 +130,9 @@ std::expected<CpuModel, std::string> CpuModel::load(const std::string& gguf_path
 #undef TAKE
     if (auto r = ld.check_all_used(); !r) return std::unexpected(r.error());
 
-    m.n_ctx_ = std::min(c.n_ctx_train, kMaxCtx);
+    if (n_ctx > c.n_ctx_train)
+        return std::unexpected(std::format("n_ctx {} exceeds the model's training context {}", n_ctx, c.n_ctx_train));
+    m.n_ctx_ = n_ctx ? n_ctx : std::min(c.n_ctx_train, kDefaultCtx);
     const size_t cache = size_t{c.n_layer} * m.n_ctx_ * KV;
     m.k_cache_.assign(cache, 0.0f);
     m.v_cache_.assign(cache, 0.0f);
@@ -150,6 +152,8 @@ std::expected<CpuModel, std::string> CpuModel::load(const std::string& gguf_path
 
 void CpuModel::reset() { n_past_ = 0; }
 
+void CpuModel::truncate(uint32_t n) { n_past_ = std::min(n, n_past_); }
+
 void CpuModel::matvec(const std::vector<float>& w, const float* x, float* y, uint32_t rows, uint32_t cols) {
     pool_->parallel_for(rows, [&](size_t begin, size_t end) {
         for (size_t r = begin; r < end; ++r) {

engine/backends/cpu/cpu_model.h

@@ -36,15 +36,22 @@ class CpuModel {
     // Loads every tensor of the GGUF. Fails if a tensor is missing, has the
     // wrong shape, has an unsupported type, or is present but unused (an
     // unused tensor means the model has semantics this code does not model).
-    static std::expected<CpuModel, std::string> load(const std::string& gguf_path, size_t n_threads = 0);
+    // n_ctx: KV cache length; 0 uses min(training context, 4096).
+    static std::expected<CpuModel, std::string> load(const std::string& gguf_path, size_t n_threads = 0,
+                                                     uint32_t n_ctx = 0);
 
     const ModelConfig& config() const { return cfg_; }
 
     // Clears the KV cache.
     void reset();
 
-    // Number of tokens in the KV cache.
+    // Number of tokens in the KV cache, and its capacity.
     uint32_t n_past() const { return n_past_; }
+    uint32_t n_ctx() const { return n_ctx_; }
+
+    // Drops cached positions from n onwards (n <= n_past()), so a new
+    // sequence sharing the first n tokens continues from there.
+    void truncate(uint32_t n);
 
     // Runs one token at position n_past() and returns the next-token logits
     // (config().n_vocab floats). Fails when the context is full.