install: send credentials embedded in a tarball URL as Basic authorization

docs/pm/cli/add.mdx

@@ -231,6 +231,8 @@ bun add zod@https://registry.npmjs.org/zod/-/zod-3.21.4.tgz
 }
 ```
 
+A tarball URL can carry credentials, such as `https://user:password@example.com/zod-3.21.4.tgz`. Bun sends them as an `Authorization: Basic` header and requests the URL without them, like npm. The URL, credentials included, is written to `package.json` and to the lockfile.
+
 ---
 
 <Add />

src/install/NetworkTask.rs

@@ -408,6 +408,40 @@ fn count_auth(header_builder: &mut HeaderBuilder, scope: &npm::registry::Scope)
     header_builder.count("npm-auth-type", "legacy");
 }
 
+/// Splits `http://user:pass@host/pkg.tgz` into `user:pass` and
+/// `http://host/pkg.tgz`. `None` when the authority has no `@`; the `@` of a
+/// scoped package in the path (`/@scope/pkg/-/pkg.tgz`) is not one.
+fn split_url_userinfo(url: &[u8]) -> Option<(&[u8], Box<[u8]>)> {
+    let authority_start = strings::index_of(url, b"://")? + b"://".len();
+    let rest = &url[authority_start..];
+    let authority = &rest[..strings::index_of_any(rest, b"/?#").unwrap_or(rest.len())];
+    let at = strings::last_index_of_char(authority, b'@')?;
+
+    let mut without_userinfo = Vec::with_capacity(url.len() - (at + 1));
+    without_userinfo.extend_from_slice(&url[..authority_start]);
+    without_userinfo.extend_from_slice(&rest[at + 1..]);
+    Some((&rest[..at], without_userinfo.into_boxed_slice()))
+}
+
+/// `Basic base64(userinfo)`, the header npm sends for credentials embedded in a
+/// tarball URL: minipass-fetch (`getNodeRequestOptions` in `lib/request.js`)
+/// hands the URL's `username:password` to node's `auth` option as is, so
+/// nothing is percent-decoded here either, and a userinfo without a `:` is a
+/// username with an empty password.
+fn basic_authorization_from_userinfo(userinfo: &[u8]) -> Vec<u8> {
+    const SCHEME: &[u8] = b"Basic ";
+    let mut user_pass = Vec::with_capacity(userinfo.len() + 1);
+    user_pass.extend_from_slice(userinfo);
+    if !strings::contains_char(userinfo, b':') {
+        user_pass.push(b':');
+    }
+    let mut value = vec![0u8; SCHEME.len() + bun_core::base64::encode_len(&user_pass)];
+    value[..SCHEME.len()].copy_from_slice(SCHEME);
+    let encoded_len = bun_core::base64::encode(&mut value[SCHEME.len()..], &user_pass);
+    value.truncate(SCHEME.len() + encoded_len);
+    value
+}
+
 #[derive(thiserror::Error, Debug, strum::IntoStaticStr)]
 pub enum ForManifestError {
     #[error("OutOfMemory")]
@@ -784,6 +818,21 @@ impl NetworkTask {
             return Err(ForTarballError::InvalidURL);
         }
 
+        // `"dep": "https://user:pass@host/dep.tgz"`: the credentials become a
+        // header, as npm sends them, and the URL is requested without them.
+        // They cannot stay in the URL: `bun_url` keeps the userinfo in `origin`,
+        // and the HTTP client compares origins to decide whether `Authorization`
+        // follows a redirect, so a redirect to the same host would lose it.
+        let url_authorization: Option<Vec<u8>> = match split_url_userinfo(&self.url_buf) {
+            Some((userinfo, url_without_userinfo)) => {
+                let value =
+                    (!userinfo.is_empty()).then(|| basic_authorization_from_userinfo(userinfo));
+                self.url_buf = url_without_userinfo;
+                value
+            }
+            None => None,
+        };
+
         // Only attach the registry `Authorization` header when the tarball URL
         // origin matches the configured registry scope origin. The npm manifest
         // is registry-controlled, so a malicious registry could otherwise point
@@ -815,9 +864,23 @@ impl NetworkTask {
             count_auth(&mut header_builder, scope);
         }
 
+        // Same precedence as npm, where node derives `Authorization` from the
+        // URL only when the request does not carry one already: credentials
+        // configured for the registry win over the ones embedded in the URL.
+        let url_authorization = match url_authorization {
+            Some(value) if header_builder.header_count == 0 => {
+                header_builder.count("Authorization", &value);
+                Some(value)
+            }
+            _ => None,
+        };
+
         let header_buf: &'static [u8] = if header_builder.header_count > 0 {
             header_builder.allocate()?;
-            append_auth(&mut header_builder, scope);
+            match &url_authorization {
+                Some(value) => header_builder.append("Authorization", value),
+                None => append_auth(&mut header_builder, scope),
+            }
             debug_assert_eq!(header_builder.content.len, header_builder.content.cap);
             self.header_buf = header_builder.content.move_to_slice();
             // SAFETY: `self.header_buf` outlives the request; it is freed when the slot returns to the pool.

test/cli/install/bun-install.test.ts

@@ -854,6 +854,223 @@ describe.concurrent("bun-install", () => {
     expect(exitCode).toBe(0);
   });
 
+  // A tarball URL with credentials in it is downloaded the way npm downloads
+  // it: the userinfo becomes `Authorization: Basic base64(user:pass)` and the
+  // request goes to the URL without it (`NetworkTask::for_tarball`).
+  describe.concurrent("credentials embedded in a tarball URL", () => {
+    const tgz = join(import.meta.dir, "registry", "packages", "no-deps", "no-deps-1.0.0.tgz");
+    const tarballPath = "/cdn/no-deps-1.0.0.tgz";
+    const basic = (userPass: string) => `Basic ${Buffer.from(userPass).toString("base64")}`;
+    const installed = {
+      stdout: expect.stringContaining("1 package installed"),
+      stderr: expect.stringContaining("Saved lockfile"),
+      exitCode: 0,
+    };
+
+    type Received = { url: string; authorization: string | null };
+
+    function recording(received: Received[], handler: (req: Request, server: { port: number }) => Response) {
+      return (req: Request, server: { port: number }) => {
+        received.push({ url: req.url, authorization: req.headers.get("authorization") });
+        return handler(req, server);
+      };
+    }
+
+    // Serves `tgz` to `.tgz` requests carrying exactly `authorization` and
+    // answers 401 to the others. A request under `/redirect/` is first
+    // redirected to `redirectTo`, or to the same file under `/cdn/`.
+    function serveTarball(received: Received[], authorization: string | null, redirectTo?: string) {
+      return Bun.serve({
+        port: 0,
+        hostname: "127.0.0.1",
+        fetch: recording(received, (req, server) => {
+          const { pathname } = new URL(req.url);
+          if (pathname.startsWith("/redirect/")) {
+            const name = pathname.slice("/redirect/".length);
+            return Response.redirect(redirectTo ?? `http://127.0.0.1:${server.port}/cdn/${name}`, 302);
+          }
+          if (req.headers.get("authorization") !== authorization) {
+            return new Response("unauthorized", { status: 401 });
+          }
+          return new Response(file(tgz));
+        }),
+      });
+    }
+
+    // `bun install` of a project whose only dependency `no-deps` is `dependency`.
+    async function install(dependency: string, files: Record<string, string> = {}, args: string[] = []) {
+      using dir = tempDir("tarball-url-credentials", {
+        "package.json": JSON.stringify({ name: "app", version: "1.0.0", dependencies: { "no-deps": dependency } }),
+        ...files,
+      });
+      await using proc = spawn({
+        cmd: [bunExe(), "install", ...args],
+        cwd: String(dir),
+        env: { ...env, BUN_INSTALL_CACHE_DIR: join(String(dir), ".cache") },
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+      return { stdout, stderr, exitCode };
+    }
+
+    // Each row is the userinfo of the dependency URL and the `user:pass` the
+    // header must encode. It is sent as written: like npm (checked with npm
+    // 11), a missing password is sent as an empty one and percent-encoding is
+    // left alone. npm would percent-encode the second colon of the last row
+    // because it serializes the URL first.
+    it.each([
+      ["a username and a password", "carol:s3cret", "carol:s3cret", []],
+      ["a username and a password, isolated linker", "carol:s3cret", "carol:s3cret", ["--linker", "isolated"]],
+      ["a username only", "carol", "carol:", []],
+      ["a password only", ":s3cret", ":s3cret", []],
+      ["a percent-encoded password", "carol:s3%40cret", "carol:s3%40cret", []],
+      ["a password containing a colon", "carol:s3:cret", "carol:s3:cret", []],
+    ])("sends %s as Basic authorization", async (_, userinfo, userPass, args) => {
+      const authorization = basic(userPass);
+      const received: Received[] = [];
+      await using server = serveTarball(received, authorization);
+
+      const result = await install(`http://${userinfo}@127.0.0.1:${server.port}${tarballPath}`, {}, args);
+
+      expect({ received, ...result }).toEqual({
+        received: [{ url: `http://127.0.0.1:${server.port}${tarballPath}`, authorization }],
+        ...installed,
+      });
+    });
+
+    it("does not take the @ of a scoped package path for credentials", async () => {
+      const received: Received[] = [];
+      await using server = serveTarball(received, null);
+      const scopedPath = "/@scope/no-deps/-/no-deps-1.0.0.tgz";
+
+      const result = await install(`http://127.0.0.1:${server.port}${scopedPath}`);
+
+      expect({ received, ...result }).toEqual({
+        received: [{ url: `http://127.0.0.1:${server.port}${scopedPath}`, authorization: null }],
+        ...installed,
+      });
+    });
+
+    it("keeps the credentials across a redirect within the host", async () => {
+      const received: Received[] = [];
+      await using server = serveTarball(received, basic("carol:s3cret"));
+
+      const result = await install(`http://carol:s3cret@127.0.0.1:${server.port}/redirect/no-deps-1.0.0.tgz`);
+
+      expect({ received, ...result }).toEqual({
+        received: [
+          { url: `http://127.0.0.1:${server.port}/redirect/no-deps-1.0.0.tgz`, authorization: basic("carol:s3cret") },
+          { url: `http://127.0.0.1:${server.port}${tarballPath}`, authorization: basic("carol:s3cret") },
+        ],
+        ...installed,
+      });
+    });
+
+    it("drops the credentials on a redirect to another host", async () => {
+      // The same machine, reached under a hostname other than the one the
+      // credentials were written for. This host serves the tarball regardless.
+      const otherHostReceived: Received[] = [];
+      await using otherHost = Bun.serve({
+        port: 0,
+        fetch: recording(otherHostReceived, () => new Response(file(tgz))),
+      });
+      const received: Received[] = [];
+      await using server = serveTarball(received, null, `http://localhost:${otherHost.port}${tarballPath}`);
+
+      const result = await install(`http://carol:s3cret@127.0.0.1:${server.port}/redirect/no-deps-1.0.0.tgz`);
+
+      expect({ received, otherHostReceived, ...result }).toEqual({
+        received: [
+          { url: `http://127.0.0.1:${server.port}/redirect/no-deps-1.0.0.tgz`, authorization: basic("carol:s3cret") },
+        ],
+        otherHostReceived: [{ url: `http://localhost:${otherHost.port}${tarballPath}`, authorization: null }],
+        ...installed,
+      });
+    });
+
+    it("reports a rejected download by the URL without the credentials", async () => {
+      const received: Received[] = [];
+      await using server = serveTarball(received, basic("carol:s3cret"));
+
+      const result = await install(`http://carol:wrong@127.0.0.1:${server.port}${tarballPath}`);
+
+      expect({ received, ...result }).toEqual({
+        received: [{ url: `http://127.0.0.1:${server.port}${tarballPath}`, authorization: basic("carol:wrong") }],
+        stdout: expect.stringContaining("bun install v1."),
+        stderr: expect.stringContaining(`error: GET http://127.0.0.1:${server.port}${tarballPath} - 401`),
+        exitCode: 1,
+      });
+    });
+
+    // A registry whose manifest puts credentials into `dist.tarball`. As with
+    // npm, the credentials configured for the registry take precedence; the
+    // URL's are used when the registry has none.
+    describe.concurrent("in the dist.tarball URL of a registry manifest", () => {
+      const token = "registry-token";
+      const distPath = "/no-deps/-/no-deps-1.0.0.tgz";
+
+      function serveRegistry(received: Received[], tarballAuthorization: string | null) {
+        return Bun.serve({
+          port: 0,
+          hostname: "127.0.0.1",
+          fetch: recording(received, (req, server) => {
+            const { pathname } = new URL(req.url);
+            if (pathname === "/no-deps") {
+              return Response.json({
+                name: "no-deps",
+                "dist-tags": { latest: "1.0.0" },
+                versions: {
+                  "1.0.0": {
+                    name: "no-deps",
+                    version: "1.0.0",
+                    dist: { tarball: `http://dist:d1st@127.0.0.1:${server.port}${distPath}` },
+                  },
+                },
+              });
+            }
+            if (pathname === distPath && req.headers.get("authorization") === tarballAuthorization) {
+              return new Response(file(tgz));
+            }
+            return new Response("unauthorized", { status: 401 });
+          }),
+        });
+      }
+
+      it("sends the registry's credentials when it has some", async () => {
+        const received: Received[] = [];
+        await using registry = serveRegistry(received, `Bearer ${token}`);
+
+        const result = await install("1.0.0", {
+          ".npmrc": `registry=http://127.0.0.1:${registry.port}/\n//127.0.0.1:${registry.port}/:_authToken=${token}\n`,
+        });
+
+        expect({ received, ...result }).toEqual({
+          received: [
+            { url: `http://127.0.0.1:${registry.port}/no-deps`, authorization: `Bearer ${token}` },
+            { url: `http://127.0.0.1:${registry.port}${distPath}`, authorization: `Bearer ${token}` },
+          ],
+          ...installed,
+        });
+      });
+
+      it("sends the URL's credentials when the registry has none", async () => {
+        const received: Received[] = [];
+        await using registry = serveRegistry(received, basic("dist:d1st"));
+
+        const result = await install("1.0.0", { ".npmrc": `registry=http://127.0.0.1:${registry.port}/\n` });
+
+        expect({ received, ...result }).toEqual({
+          received: [
+            { url: `http://127.0.0.1:${registry.port}/no-deps`, authorization: null },
+            { url: `http://127.0.0.1:${registry.port}${distPath}`, authorization: basic("dist:d1st") },
+          ],
+          ...installed,
+        });
+      });
+    });
+  });
+
   it("--silent suppresses verbose output even when RUNNER_DEBUG is set", async () => {
     using dir = tempDir("install-silent-verbose", {
       "package.json": JSON.stringify({ name: "app", dependencies: {} }),