http: replace picohttpparser with a Rust parser; classify headers once and build fetch Response headers lazily

.github/workflows/miri.yml

@@ -19,6 +19,7 @@ on:
       - "src/http_types/**"
       - "src/md/**"
       - "src/paths/**"
+      - "src/picohttp/**"
       - "src/ptr/**"
       - "src/resolve_builtins/**"
       - "src/shell_parser/**"

.gitignore

@@ -159,7 +159,6 @@ codegen-for-zig-team.tar.gz
 /src/deps/lol*html
 /src/deps/ls*hpack
 /src/deps/mimalloc
-/src/deps/picohttpparser
 /src/deps/tinycc
 /src/deps/WebKit
 /src/deps/zig

CLAUDE.md

@@ -147,7 +147,7 @@ The Rust side is a Cargo workspace of ~200 crates rooted at `Cargo.toml`. The ke
 
 #### Vendored Dependencies (`vendor/`)
 
-Third-party C/C++ libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), picohttpparser, tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`.
+Third-party C/C++ libraries are vendored locally and can be read from disk (not git submodules): boringssl (TLS/crypto), brotli, cares (async DNS), hdrhistogram, highway (SIMD), libarchive (tar/zip), libdeflate, libuv (Windows event loop), lolhtml (HTML rewriter), lshpack (HTTP/2 HPACK), lsqpack + lsquic (HTTP/3), mimalloc (allocator), nodejs (headers), tinycc (FFI JIT, fork: oven-sh/tinycc), WebKit (JavaScriptCore), zlib (zlib-ng), zstd. Build configuration for these is in `scripts/build/deps/*.ts`.
 
 ### JavaScript Class Implementation (C++)
 

LICENSE.md

@@ -28,7 +28,6 @@ Bun statically links these libraries:
 | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT |
 | [`lsquic`](https://github.com/litespeedtech/lsquic) | MIT (portions derived from [Chromium proto-quic](https://github.com/litespeedtech/lsquic/blob/master/LICENSE.chrome), BSD 3-Clause) |
 | [`mimalloc`](https://github.com/microsoft/mimalloc) | MIT |
-| [`picohttp`](https://github.com/h2o/picohttpparser) | dual-licensed under the Perl License or the MIT License |
 | [`zstd`](https://github.com/facebook/zstd) | dual-licensed under the BSD License or GPLv2 license |
 | [`simdutf`](https://github.com/simdutf/simdutf) | Apache 2.0 |
 | [`tinycc`](https://github.com/tinycc/tinycc) | LGPL v2.1 |
@@ -45,6 +44,7 @@ Bun statically links these libraries:
 | [`highway`](https://github.com/google/highway) | Apache 2.0 |
 | [`uucode`](https://github.com/jacobsandlund/uucode) | MIT |
 | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets) | Apache 2.0 licensed |
+| HTTP/1.1 chunked-encoding decoder derived from [`picohttpparser`](https://github.com/h2o/picohttpparser) | MIT licensed |
 | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed |
 | `__cxa_thread_atexit` fallback from [LLVM libc++abi](https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/libcxxabi/src/cxa_thread_atexit.cpp) | Apache 2.0 with LLVM exception |
 

bench/fetch/response-headers-server.mjs

@@ -0,0 +1,42 @@
+// Serves a 2-byte body under one of three response-header profiles, keyed by
+// path: /small (2 headers), /nginx (14, API-gateway style), /cookies (long
+// Location/Set-Cookie values). Pair with response-headers.mjs; run it in its
+// own process (ideally pinned to other cores) so it doesn't share a CPU with
+// the client:  PORT=4001 bun bench/fetch/response-headers-server.mjs
+const sets = {
+  small: { "Content-Type": "text/plain", "X-Id": "1" },
+  nginx: {
+    "Server": "nginx/1.25.3",
+    "Content-Type": "application/json; charset=utf-8",
+    "Vary": "Accept-Encoding",
+    "Cache-Control": "private, max-age=0, no-cache, no-store, must-revalidate",
+    "ETag": 'W/"bc55-7Zci8Yc4Bq3vJk8pQ0h5nX0m9sE"',
+    "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload",
+    "X-Content-Type-Options": "nosniff",
+    "X-Frame-Options": "SAMEORIGIN",
+    "X-Request-Id": "3f1c9a7e-5b2d-4c8e-9f0a-1b2c3d4e5f60",
+    "Access-Control-Allow-Origin": "*",
+    "Access-Control-Expose-Headers": "X-Request-Id, ETag, Link",
+    "X-RateLimit-Limit": "5000",
+    "X-RateLimit-Remaining": "4987",
+    "X-RateLimit-Reset": "1723025489",
+  },
+  cookies: {
+    "Location":
+      "https://accounts.example.com/signin/v2/identifier?continue=https%3A%2F%2Fmail.example.com%2Fmail%2F&service=mail&flowName=GlifWebSignIn&flowEntry=ServiceLogin",
+    "Set-Cookie":
+      "__Host-SESSION=CgQIARAB.eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=2592000",
+    "Content-Type": "text/html; charset=UTF-8",
+    "Alt-Svc": 'h3=":443"; ma=2592000,h3-29=":443"; ma=2592000',
+  },
+};
+const responses = Object.fromEntries(Object.entries(sets).map(([k, h]) => [k, { headers: new Headers(h) }]));
+const server = Bun.serve({
+  port: Number(process.env.PORT || 0),
+  reusePort: true,
+  fetch(req) {
+    const kind = req.url.slice(req.url.lastIndexOf("/") + 1) || "nginx";
+    return new Response("ok", responses[kind]);
+  },
+});
+console.log(server.port);

bench/fetch/response-headers.mjs

@@ -0,0 +1,47 @@
+// Client CPU per fetch() against response-headers-server.mjs.
+//   bun bench/fetch/response-headers.mjs <port> [small|nginx|cookies] [body|get|iter] [requests] [concurrency]
+// mode: body = read the body only; get = also headers.get(); iter = also iterate all headers.
+// Reports requests/s and this process's CPU time per request (user+sys across
+// all threads), which is the number to compare between builds when the server
+// is the throughput bottleneck.
+const [port, kind = "nginx", mode = "body", total = 200000, conc = 64] = process.argv.slice(2);
+const url = `http://127.0.0.1:${port}/${kind}`;
+const N = Number(total),
+  C = Number(conc);
+let sink = 0;
+async function one() {
+  const r = await fetch(url);
+  if (mode === "get") sink += r.headers.get("content-type").length;
+  else if (mode === "iter") for (const [k, v] of r.headers) sink += k.length + v.length;
+  const b = await r.arrayBuffer();
+  sink += b.byteLength;
+}
+// warmup
+await Promise.all(Array.from({ length: C }, one));
+const cpu0 = process.cpuUsage();
+const t0 = performance.now();
+let started = 0;
+async function worker() {
+  while (started < N) {
+    started++;
+    await one();
+  }
+}
+await Promise.all(Array.from({ length: C }, worker));
+const t = performance.now() - t0;
+const cpu = process.cpuUsage(cpu0);
+const cpuMs = (cpu.user + cpu.system) / 1000;
+console.log(
+  JSON.stringify({
+    kind,
+    mode,
+    N,
+    C,
+    ms: +t.toFixed(0),
+    rps: Math.round(N / (t / 1000)),
+    cpu_us_per_req: +((cpuMs * 1000) / N).toFixed(2),
+    user_ms: Math.round(cpu.user / 1000),
+    sys_ms: Math.round(cpu.system / 1000),
+    sink,
+  }),
+);

docs/project/license.mdx

@@ -30,7 +30,6 @@ Bun statically links these libraries:
 | [`libarchive`](https://github.com/libarchive/libarchive)                                                                                         | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING)             |
 | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api)                                                                           | BSD 3-Clause                                                                                 |
 | [`mimalloc`](https://github.com/microsoft/mimalloc)                                                                                              | MIT                                                                                          |
-| [`picohttp`](https://github.com/h2o/picohttpparser)                                                                                              | dual-licensed under the Perl License or the MIT License                                      |
 | [`zstd`](https://github.com/facebook/zstd)                                                                                                       | dual-licensed under the BSD License or GPLv2 license                                         |
 | [`simdutf`](https://github.com/simdutf/simdutf)                                                                                                  | Apache 2.0                                                                                   |
 | [`tinycc`](https://github.com/tinycc/tinycc)                                                                                                     | LGPL v2.1                                                                                    |
@@ -42,6 +41,7 @@ Bun statically links these libraries:
 | [`libuv`](https://github.com/libuv/libuv) (on Windows)                                                                                           | MIT                                                                                          |
 | [`libdeflate`](https://github.com/ebiggers/libdeflate)                                                                                           | MIT                                                                                          |
 | A fork of [`uWebsockets`](https://github.com/jarred-sumner/uwebsockets)                                                                          | Apache 2.0 licensed                                                                          |
+| HTTP/1.1 chunked-encoding decoder derived from [`picohttpparser`](https://github.com/h2o/picohttpparser)                                         | MIT licensed                                                                                 |
 | Parts of [Tigerbeetle's IO code](https://github.com/tigerbeetle/tigerbeetle/blob/532c8b70b9142c17e07737ab6d3da68d7500cbca/src/io/windows.zig#L1) | Apache 2.0 licensed                                                                          |
 
 ## Polyfills

scripts/build/bun.ts

@@ -189,9 +189,7 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     }
   }
 
-  // Collect all dep lib paths, include dirs, output stamps, and directly-
-  // compiled source files (deps like picohttpparser that provide .c files
-  // instead of a .a — we compile those alongside bun's own sources).
+  // Collect all dep lib paths, include dirs, and output stamps.
   const depLibs: string[] = [];
   const depObjects: string[] = [];
   const depIncludes: string[] = [];
@@ -325,9 +323,6 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     noPchSources.add(rescleBinding);
   }
 
-  // Deps with provides.sources compiled in the loop below so each dep's
-  // phony can point at its own .o files.
-
   // Codegen .cpp files — compiled like regular sources.
   cxxSources.push(...codegen.cppSources);
   cxxSources.push(...codegen.bindgenV2Cpp);
@@ -387,16 +382,15 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
 
   // Compile all .c files. No PCH — dep signal applied directly.
   const cObjects: string[] = [];
-  const compileC = (src: string): string => {
-    const obj = cc(n, cfg, src, {
-      flags: cFlagsFull,
-      implicitInputs: depHeaderSignal,
-      orderOnlyInputs: codegenOrderOnly,
-    });
-    cObjects.push(obj);
-    return obj;
-  };
-  for (const src of cSources) compileC(src);
+  for (const src of cSources) {
+    cObjects.push(
+      cc(n, cfg, src, {
+        flags: cFlagsFull,
+        implicitInputs: depHeaderSignal,
+        orderOnlyInputs: codegenOrderOnly,
+      }),
+    );
+  }
 
   // InternalModuleRegistryConstants.S — `.incbin`s the bundled JS module sources
   // so InternalModuleRegistry.cpp sees a tiny {offset, length} table instead of
@@ -411,16 +405,6 @@ export function emitBun(n: Ninja, cfg: Config, sources: Sources): BunOutput {
     }),
   );
 
-  // Deps that contribute source files for bun to compile directly (via
-  // provides.sources) instead of building a lib. Compile them here with
-  // bun's full flag set and give each a phony so `--target <name>` builds
-  // its .o files. libs.length === 0 guard: deps with a build step already
-  // got a phony in resolveDep — don't emit a duplicate.
-  for (const d of deps) {
-    if (d.sources.length === 0 || d.libs.length > 0) continue;
-    n.phony(d.name, d.sources.map(compileC));
-  }
-
   // Dep objects (when !cfg.archiveDeps) are linked alongside bun's own
   // objects — same response file, same archive in cpp-only mode. With
   // cfg.archiveDeps they live in depLibs as .a files instead.

scripts/build/deps/index.ts

@@ -28,7 +28,6 @@ import { lsqpack } from "./lsqpack.ts";
 import { lsquic } from "./lsquic.ts";
 import { mimalloc } from "./mimalloc.ts";
 import { nodejsHeaders } from "./nodejs-headers.ts";
-import { picohttpparser } from "./picohttpparser.ts";
 import { sqlite } from "./sqlite.ts";
 import { tinycc } from "./tinycc.ts";
 import { webkit } from "./webkit.ts";
@@ -43,8 +42,7 @@ import { zstd } from "./zstd.ts";
  * boringssl near the end — many things depend on crypto/ssl symbols.
  */
 export const allDeps: readonly Dependency[] = [
-  // Header-only / source-only first — no link order concerns.
-  picohttpparser,
+  // Header-only first — no link order concerns.
   nodejsHeaders,
 
   zlib,
@@ -95,7 +93,6 @@ export {
   lsquic,
   mimalloc,
   nodejsHeaders,
-  picohttpparser,
   sqlite,
   tinycc,
   webkit,

scripts/build/flags.ts

@@ -1531,7 +1531,6 @@ export function bunIncludes(cfg: Config): string[] {
     join(cwd, "src/uws_sys"),
     codegenDir,
     vendorDir,
-    join(vendorDir, "picohttpparser"),
     join(vendorDir, "zlib"),
     // NODEJS_HEADERS_PATH comes from the nodejs dep; added separately
   ];

scripts/build/source.ts

@@ -390,16 +390,6 @@ export interface Provides {
   includes: string[];
   /** Preprocessor defines to add to bun's compilation. */
   defines?: string[];
-  /**
-   * Source files (relative to the SOURCE dir) that bun compiles directly
-   * into its own binary — no nested build producing a `.a`. Declared as
-   * implicit outputs of the fetch rule so ninja knows where they come from;
-   * bun.ts adds them to its C/C++ source lists.
-   *
-   * Most deps provide `.a` files via `libs`. This is for the rare case of
-   * a single-file dep with no build system (picohttpparser: one .c file).
-   */
-  sources?: string[];
 }
 
 /**
@@ -483,11 +473,6 @@ export interface ResolvedDep {
   /** Absolute include paths for -I flags. */
   includes: string[];
   defines: string[];
-  /**
-   * Absolute paths to .c/.cpp files bun compiles directly (from
-   * Provides.sources). Empty for most deps — they provide .a files.
-   */
-  sources: string[];
   /**
    * The final build output(s). Use these as implicit inputs on anything
    * downstream that needs this dep built first.
@@ -730,15 +715,10 @@ export function resolveDep(
   // we don't want patches changing between emitFetch and the hash check.
   const patches = dep.patches === undefined ? [] : typeof dep.patches === "function" ? dep.patches(cfg) : dep.patches;
 
-  // Sources bun compiles directly (from Provides.sources). Resolved to
-  // absolute paths for (a) the ResolvedDep return and (b) declaring as
-  // implicit outputs of fetch so ninja knows where they come from.
-  const resolvedSources = (provides.sources ?? []).map(s => resolve(srcDir, s));
-
-  // DirectBuild sources are ALSO compiled in our ninja graph, so they need
-  // the same implicit-output-of-fetch treatment. Include the codegen tool
-  // source, its input, and any HeaderSubst templates — all read at build
-  // time from the fetched tree.
+  // DirectBuild sources are compiled in our ninja graph, so they must be
+  // declared as implicit outputs of fetch so ninja knows where they come
+  // from. Include the codegen tool source, its input, and any HeaderSubst
+  // templates — all read at build time from the fetched tree.
   const directSources: string[] = [];
   if (buildSpec.kind === "direct") {
     for (const s of buildSpec.sources) {
@@ -762,7 +742,7 @@ export function resolveDep(
   //   (CMakeLists.txt) as the stamp. Editing it → reconfigure.
   let sourceStamp: string;
   if (source.kind === "github-archive") {
-    sourceStamp = emitFetch(n, cfg, dep.name, source, patches, [...resolvedSources, ...directSources]);
+    sourceStamp = emitFetch(n, cfg, dep.name, source, patches, directSources);
   } else {
     // Local/in-tree: no .ref to write. Use the build system's manifest file
     // as the stamp — touching it triggers reconfigure/rebuild.
@@ -842,9 +822,7 @@ export function resolveDep(
     // are link inputs, not include-order dependencies).
     outputs = result.headerOutputs;
   } else {
-    // No build step. Source stamp is the only output. For deps with
-    // provides.sources (picohttpparser), emitBun adds a phony pointing at
-    // the compiled .o files so `--target <name>` actually compiles them.
+    // No build step (header-only). Source stamp is the only output.
     libs = [];
     outputs = [sourceStamp];
   }
@@ -868,7 +846,6 @@ export function resolveDep(
     objects,
     includes,
     defines: provides.defines ?? [],
-    sources: resolvedSources,
     outputs,
   };
 }
@@ -933,7 +910,7 @@ export function computeDepLibs(cfg: Config, dep: Dependency): string[] {
     return [resolve(buildDir, `${cfg.libPrefix}${dep.name}${cfg.libSuffix}`)];
   }
 
-  // none: no libs (header-only or directly-compiled sources).
+  // none: no libs (header-only).
   return [];
 }
 
@@ -974,7 +951,7 @@ function emitFetch(
 
   n.build({
     outputs: [refStamp],
-    // Source files bun compiles directly (picohttpparser.c). Declaring
+    // Source files bun compiles directly (DirectBuild deps). Declaring
     // them as outputs tells ninja "fetch creates these" — otherwise ninja
     // errors "missing and no known rule to make it" on fresh checkouts.
     ...(compiledSources.length > 0 && { implicitOutputs: compiledSources }),
@@ -1062,7 +1039,6 @@ function emitPrebuilt(
     objects: [],
     includes,
     defines: provides.defines ?? [],
-    sources: [],
     outputs,
   };
 }