From edd0e5e79535c27ff6ba4e3f041a4771f3d5ef7d Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Sat, 4 Jul 2026 20:53:29 +0530
Subject: [PATCH 01/19] feat(oss-db): add automated build script

Adds floss/qs/db/data/oss/build_oss_db.py that automates the vcpkg + jh
pipeline for building OSS string databases:

  1. install static libraries via vcpkg
  2. extract string features (and function names) via jh
  3. convert to JSONL and compress with gzip

The converter deduplicates within each library (one metadata object per
unique string) and emits each library as its own .jsonl.gz file, matching
the schema expected by floss.qs.db.oss.OpenSourceStringDatabase.

The script accepts a JSON config via --config or env vars and supports
selecting individual libraries, controlling output directory, and
continuing past per-library build failures.
---
 floss/qs/db/data/oss/build_oss_db.py | 972 +++++++++++++++++++++++++++
 floss/qs/db/data/oss/readme.md       |  16 +
 2 files changed, 988 insertions(+)
 create mode 100755 floss/qs/db/data/oss/build_oss_db.py

diff --git a/floss/qs/db/data/oss/build_oss_db.py b/floss/qs/db/data/oss/build_oss_db.py
new file mode 100755
index 000000000..418217589
--- /dev/null
+++ b/floss/qs/db/data/oss/build_oss_db.py
@@ -0,0 +1,972 @@
+#!/usr/bin/env python3
+"""
+Build OSS string databases from vcpkg static libraries.
+
+This script automates the "vcpkg & jh" technique described in readme.md:
+
+  1. install static libraries via vcpkg
+  2. extract string features (and function names) via jh
+  3. convert to JSONL and compress with gzip
+
+It is intentionally modular so the underlying extractor (jh today) can be
+swapped for a more minimal tool later without rewriting the orchestration.
+
+After every library is parsed, strings that appear in two or more libraries
+are removed from *all* of those libraries. A shared string does not uniquely
+identify a library, so attributing it to any specific one would be a false
+positive at query time.
+
+By default, the script merges newly-extracted entries with any pre-existing
+``.jsonl.gz`` in the output directory rather than replacing them. This makes
+it safe to incrementally add a library (e.g. ``--libraries newlib``) without
+losing the others, and safe to bump one library's version while leaving its
+previously-committed entries around. Libraries that were not rebuilt still
+participate in the cross-library dedup pass and are rewritten if the shared
+string set changes.
+
+Example usage:
+
+  # Default: build the top 5 largest databases using x64-windows-static.
+  python build_oss_db.py --lancelot-dir ~/Projects/Mandiant/lancelot
+
+  # Build a specific library with a different triplet.
+  python build_oss_db.py --triplet x64-mingw-static --libraries openssl
+
+  # Inside a container where vcpkg/jh are already on PATH.
+  python build_oss_db.py --triplet x64-linux
+
+Environment variables (used as defaults when CLI flags are omitted):
+
+  VCPKG_ROOT          root directory of a vcpkg installation
+  LANCELOT_DIR        directory containing the lancelot source (for building jh)
+  JH_PATH             path to an existing jh binary
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+import gzip
+import json
+import shutil
+import logging
+import pathlib
+import argparse
+import subprocess
+from typing import Set, Dict, List, Tuple, Iterable, Optional
+from dataclasses import field, dataclass
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+)
+logger = logging.getLogger("build_oss_db")
+
+
+# Fallback library list when neither --libraries nor libraries.json is
+# provided. The actual production list lives in libraries.json.
+DEFAULT_LIBRARIES: List[str] = [
+    "openssl",
+    "sqlite3",
+    "curl",
+    "mbedtls",
+    "jemalloc",
+]
+
+# Matches the values documented in readme.md.
+DEFAULT_TRIPLET = "x64-windows-static"
+DEFAULT_COMPILER = "msvc143"
+DEFAULT_PROFILE = "release"
+
+
+class BuildError(Exception):
+    """Raised when a single library cannot be built; the caller decides whether to abort or continue."""
+
+
+class UnsupportedPlatformError(BuildError):
+    """Raised when a library does not support the target triplet/platform."""
+
+
+@dataclass(frozen=True)
+class BuildConfig:
+    triplet: str
+    compiler: str
+    profile: str
+    libraries: List[str]
+    output_dir: pathlib.Path
+    vcpkg_root: Optional[pathlib.Path]
+    jh_path: Optional[pathlib.Path]
+    lancelot_dir: Optional[pathlib.Path]
+    emit_function_names: bool = True
+    deduplicate: bool = True
+    continue_on_error: bool = False
+
+
+@dataclass
+class LibraryMetrics:
+    library: str
+    version: str
+    triplet: str
+    num_objects: int = 0
+    num_functions: int = 0
+    num_string_entries: int = 0
+    num_function_name_entries: int = 0
+    num_raw_entries: int = 0
+    num_shared_removed: int = 0
+    total_entries: int = 0
+    duration_seconds: float = 0.0
+    error: Optional[str] = None
+
+    def as_dict(self) -> dict:
+        return {
+            "library": self.library,
+            "version": self.version,
+            "triplet": self.triplet,
+            "num_objects": self.num_objects,
+            "num_functions": self.num_functions,
+            "num_string_entries": self.num_string_entries,
+            "num_function_name_entries": self.num_function_name_entries,
+            "num_raw_entries": self.num_raw_entries,
+            "num_shared_removed": self.num_shared_removed,
+            "total_entries": self.total_entries,
+            "duration_seconds": round(self.duration_seconds, 2),
+            "error": self.error,
+        }
+
+
+def run(
+    cmd: List[str],
+    cwd: Optional[pathlib.Path] = None,
+    check: bool = True,
+    env: Optional[dict] = None,
+) -> subprocess.CompletedProcess:
+    """Run a subprocess and return its output."""
+    logger.debug("running: %s", " ".join(cmd))
+    merged_env = os.environ.copy()
+    if env:
+        merged_env.update(env)
+    result = subprocess.run(
+        cmd,
+        cwd=str(cwd) if cwd else None,
+        env=merged_env,
+        text=True,
+        capture_output=True,
+    )
+    if check and result.returncode != 0:
+        raise subprocess.CalledProcessError(
+            result.returncode,
+            cmd,
+            output=result.stdout,
+            stderr=result.stderr,
+        )
+    return result
+
+
+class Vcpkg:
+    """Thin wrapper around a vcpkg installation."""
+
+    def __init__(self, vcpkg_root: Optional[pathlib.Path] = None):
+        self.exe = self._find_executable(vcpkg_root)
+        self.root = self._resolve_root(vcpkg_root)
+        self.installed_dir = self.root / "installed"
+        self.info_dir = self.installed_dir / "vcpkg" / "info"
+
+    def _find_executable(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path:
+        # 1. Executable bundled inside the provided root.
+        if vcpkg_root:
+            for name in ("vcpkg.exe", "vcpkg"):
+                candidate = vcpkg_root / name
+                if candidate.exists():
+                    return candidate.resolve()
+
+        # 2. Executable on PATH.
+        exe = shutil.which("vcpkg")
+        if exe:
+            return pathlib.Path(exe).resolve()
+
+        # 3. Executable inside VCPKG_ROOT.
+        env_root = os.environ.get("VCPKG_ROOT")
+        if env_root:
+            for name in ("vcpkg.exe", "vcpkg"):
+                candidate = pathlib.Path(env_root) / name
+                if candidate.exists():
+                    return candidate.resolve()
+
+        raise FileNotFoundError("vcpkg not found. Set VCPKG_ROOT or pass --vcpkg-root.")
+
+    def _resolve_root(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path:
+        if vcpkg_root:
+            return vcpkg_root.resolve()
+
+        env_root = os.environ.get("VCPKG_ROOT")
+        if env_root:
+            return pathlib.Path(env_root).resolve()
+
+        # The executable normally lives at <vcpkg-root>/vcpkg.
+        return self.exe.parent.resolve()
+
+    def install(self, library: str, triplet: str) -> None:
+        """Install a library for the given triplet."""
+        spec = f"{library}:{triplet}"
+        logger.info("vcpkg install %s", spec)
+        try:
+            run([str(self.exe), "install", spec])
+        except subprocess.CalledProcessError as exc:
+            output = (exc.stdout or "") + (exc.stderr or "")
+            if "is only supported on" in output:
+                raise UnsupportedPlatformError(f"{spec} is not supported on this platform") from exc
+            raise
+
+    def get_installed_version(self, library: str, triplet: str) -> str:
+        """Return the installed version string (e.g. '3.0.7#1')."""
+        result = run([str(self.exe), "list", f"{library}:{triplet}"])
+        expected_prefix = f"{library}:{triplet}"
+
+        for line in result.stdout.splitlines():
+            line = line.strip()
+            if not line or line.startswith("The following packages are"):
+                continue
+
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+
+            if parts[0] == expected_prefix:
+                return parts[1]
+
+        raise BuildError(f"could not determine installed version for {library}:{triplet}")
+
+    def find_package_libs(self, library: str, triplet: str) -> List[pathlib.Path]:
+        """Return static-library files (.lib/.a) owned by the given package."""
+        # vcpkg records installed files in <root>/installed/vcpkg/info/<package>_<version>_<triplet>.list
+        pattern = re.compile(re.escape(library) + r"_.*?_" + re.escape(triplet) + r"\.list$")
+        list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)]
+
+        if not list_files:
+            # Fallback: scan the whole lib directory. This is less precise but works
+            # when the .list file cannot be located.
+            logger.warning(
+                "could not find vcpkg info .list for %s:%s; falling back to lib-dir scan",
+                library,
+                triplet,
+            )
+            return self._find_all_static_libs(triplet)
+
+        lib_paths: List[pathlib.Path] = []
+        for list_file in list_files:
+            for line in list_file.read_text().splitlines():
+                line = line.strip()
+                if not line:
+                    continue
+                if line.startswith(triplet + "/lib/"):
+                    candidate = self.installed_dir / line
+                    if candidate.suffix in (".lib", ".a"):
+                        lib_paths.append(candidate)
+
+        return sorted(set(lib_paths))
+
+    def _find_all_static_libs(self, triplet: str) -> List[pathlib.Path]:
+        lib_dir = self.installed_dir / triplet / "lib"
+        if not lib_dir.exists():
+            return []
+        return sorted(p for p in lib_dir.iterdir() if p.suffix in (".lib", ".a"))
+
+
+class JHExtractor:
+    """Wrapper around the jh binary. Builds it from source if needed."""
+
+    def __init__(
+        self,
+        jh_path: Optional[pathlib.Path] = None,
+        lancelot_dir: Optional[pathlib.Path] = None,
+    ):
+        self.jh_path = self._resolve(jh_path, lancelot_dir)
+
+    def _resolve(
+        self,
+        jh_path: Optional[pathlib.Path],
+        lancelot_dir: Optional[pathlib.Path],
+    ) -> pathlib.Path:
+        if jh_path:
+            path = pathlib.Path(jh_path).resolve()
+            if not path.exists():
+                raise FileNotFoundError(f"jh binary not found: {path}")
+            return path
+
+        env_path = os.environ.get("JH_PATH")
+        if env_path:
+            path = pathlib.Path(env_path).resolve()
+            if path.exists():
+                return path
+
+        if lancelot_dir:
+            return self._build(lancelot_dir)
+
+        env_lancelot = os.environ.get("LANCELOT_DIR")
+        if env_lancelot:
+            return self._build(pathlib.Path(env_lancelot))
+
+        exe = shutil.which("jh")
+        if exe:
+            return pathlib.Path(exe).resolve()
+
+        raise FileNotFoundError("jh not found. Provide --jh-path, --lancelot-dir, or set JH_PATH.")
+
+    def _build(self, lancelot_dir: pathlib.Path) -> pathlib.Path:
+        logger.info("building jh from %s", lancelot_dir)
+        run(
+            ["cargo", "build", "--release", "-p", "lancelot-bin"],
+            cwd=lancelot_dir,
+        )
+        exe = lancelot_dir / "target" / "release" / "jh"
+        if sys.platform == "win32":
+            exe = exe.with_suffix(".exe")
+        if not exe.exists():
+            raise FileNotFoundError(f"jh binary not found after build: {exe}")
+        return exe.resolve()
+
+    def extract(
+        self,
+        lib_path: pathlib.Path,
+        library: str,
+        version: str,
+        triplet: str,
+        compiler: str,
+        profile: str,
+    ) -> str:
+        """Run jh on a single static library and return its CSV output."""
+        cmd = [
+            str(self.jh_path),
+            triplet,
+            compiler,
+            library,
+            version,
+            profile,
+            str(lib_path),
+        ]
+        try:
+            result = run(cmd)
+        except subprocess.CalledProcessError as exc:
+            logger.error(
+                "jh failed for %s (%s): stdout=%r stderr=%r",
+                library,
+                lib_path.name,
+                exc.stdout,
+                exc.stderr,
+            )
+            raise
+        return result.stdout
+
+
+class Converter:
+    """Convert jh JSONL output into a gzip-compressed JSONL database."""
+
+    def __init__(self, emit_function_names: bool = True, deduplicate: bool = True):
+        self.emit_function_names = emit_function_names
+        self.deduplicate = deduplicate
+
+    def parse(
+        self,
+        jh_text: str,
+        library: str,
+        version: str,
+    ) -> List[dict]:
+        """Parse jh JSONL into a list of entries (within-library dedup applied if enabled)."""
+        entries: List[dict] = []
+        function_names: Set[str] = set()
+        explicit_function_names: Set[str] = set()
+
+        for line in jh_text.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as exc:
+                logger.warning("skipping malformed JSONL line: %s (%s)", line, exc)
+                continue
+
+            file_path = row.get("path")
+            function_name = row.get("function")
+            feat_type = row.get("type")
+            value = row.get("value")
+
+            if not function_name:
+                continue
+
+            function_names.add(function_name)
+
+            if feat_type == "string":
+                entries.append(
+                    {
+                        "string": value,
+                        "library_name": library,
+                        "library_version": version,
+                        "file_path": file_path,
+                        "function_name": function_name,
+                        "line_number": None,
+                    }
+                )
+            elif feat_type == "function_name":
+                # Future-proof: a minimal extractor may emit function names explicitly.
+                entries.append(
+                    {
+                        "string": value,
+                        "library_name": library,
+                        "library_version": version,
+                        "file_path": file_path,
+                        "function_name": value,
+                        "line_number": None,
+                    }
+                )
+                explicit_function_names.add(value)
+
+        # Stock jh does not emit function_name rows, so derive them from the
+        # function column. Functions without any string/number/api features
+        # will be missed unless the extractor is patched to emit them.
+        if self.emit_function_names:
+            for fn in function_names - explicit_function_names:
+                entries.append(
+                    {
+                        "string": fn,
+                        "library_name": library,
+                        "library_version": version,
+                        "file_path": None,
+                        "function_name": fn,
+                        "line_number": None,
+                    }
+                )
+
+        if self.deduplicate:
+            # Match loader semantics: one metadata object per unique string.
+            seen: dict = {}
+            for entry in entries:
+                key = entry["string"]
+                if key not in seen:
+                    seen[key] = entry
+            entries = list(seen.values())
+
+        return entries
+
+    def write(
+        self,
+        entries: List[dict],
+        output_path: pathlib.Path,
+    ) -> dict:
+        """Write entries to a gzip-compressed JSONL file. Returns counts."""
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with gzip.open(output_path, "wt", encoding="utf-8") as f:
+            for entry in entries:
+                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+        num_string_entries = sum(
+            1 for e in entries if e["function_name"] is not None and e["function_name"] != e["string"]
+        )
+        num_function_name_entries = sum(
+            1 for e in entries if e["function_name"] is not None and e["function_name"] == e["string"]
+        )
+
+        return {
+            "num_string_entries": num_string_entries,
+            "num_function_name_entries": num_function_name_entries,
+            "total_entries": len(entries),
+        }
+
+    def convert(
+        self,
+        jh_text: str,
+        library: str,
+        version: str,
+        output_path: pathlib.Path,
+    ) -> dict:
+        """Convenience wrapper: parse jh JSONL and write JSONL.gz. Returns counts."""
+        entries = self.parse(jh_text, library, version)
+        return self.write(entries, output_path)
+
+
+def count_jsonl_rows(jh_text: str) -> dict:
+    """Count objects and unique functions referenced in jh JSONL output."""
+    objects: Set[str] = set()
+    functions: Set[str] = set()
+    for line in jh_text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        file_path = row.get("path")
+        function_name = row.get("function")
+        if function_name:
+            objects.add(file_path)
+            functions.add(function_name)
+    return {"num_objects": len(objects), "num_functions": len(functions)}
+
+
+def build_library(
+    library: str,
+    config: BuildConfig,
+    vcpkg: Vcpkg,
+    jh: JHExtractor,
+    converter: Converter,
+) -> Tuple[LibraryMetrics, List[dict]]:
+    """Parse a single library's strings. Returns metrics and the deduped entries.
+
+    The returned entries are not yet written to disk; the caller is responsible
+    for cross-library deduplication and final file emission.
+    """
+    import time
+
+    start = time.time()
+    metrics = LibraryMetrics(
+        library=library,
+        version="unknown",
+        triplet=config.triplet,
+    )
+    entries: List[dict] = []
+
+    try:
+        vcpkg.install(library, config.triplet)
+        version = vcpkg.get_installed_version(library, config.triplet)
+        metrics.version = version
+
+        lib_paths = vcpkg.find_package_libs(library, config.triplet)
+        if not lib_paths:
+            logger.info(
+                "%s: no static libraries found for %s:%s (likely header-only); skipping extraction",
+                library,
+                library,
+                config.triplet,
+            )
+        else:
+            logger.info(
+                "%s: found %d static library file(s): %s",
+                library,
+                len(lib_paths),
+                ", ".join(str(p.name) for p in lib_paths),
+            )
+
+        all_jh_parts: List[str] = []
+        for lib_path in lib_paths:
+            logger.info("%s: extracting strings from %s", library, lib_path.name)
+            jh_text = jh.extract(
+                lib_path,
+                library,
+                version,
+                config.triplet,
+                config.compiler,
+                config.profile,
+            )
+            all_jh_parts.append(jh_text)
+
+        combined_jh_text = "\n".join(all_jh_parts)
+        row_counts = count_jsonl_rows(combined_jh_text)
+        metrics.num_objects = row_counts["num_objects"]
+        metrics.num_functions = row_counts["num_functions"]
+
+        entries = converter.parse(combined_jh_text, library, version)
+        metrics.num_raw_entries = len(entries)
+    except UnsupportedPlatformError as exc:
+        logger.warning("%s: skipping unsupported library (%s)", library, exc)
+    except Exception as exc:
+        logger.exception("%s: build failed", library)
+        metrics.error = f"{type(exc).__name__}: {exc}"
+    finally:
+        metrics.duration_seconds = time.time() - start
+
+    return metrics, entries
+
+
+def find_shared_strings(per_library_entries: Dict[str, List[dict]]) -> Set[str]:
+    """Return the set of strings that appear in two or more libraries.
+
+    A shared string cannot uniquely identify a library, so it is removed from
+    every library's database to avoid false-positive library attributions.
+    """
+    seen_in: Dict[str, str] = {}
+    shared: Set[str] = set()
+    for library, entries in per_library_entries.items():
+        for entry in entries:
+            string = entry["string"]
+            if string in seen_in and seen_in[string] != library:
+                shared.add(string)
+            else:
+                seen_in[string] = library
+    return shared
+
+
+def load_existing_entries(path: pathlib.Path) -> List[dict]:
+    """Load entries from an existing OSS database .jsonl.gz.
+
+    Returns an empty list if the file is missing, empty, unreadable, or does
+    not contain entries in the expected schema. New keys are silently dropped;
+    missing keys are filled with None so the entries can be merged uniformly.
+    """
+    if not path.exists():
+        return []
+    try:
+        raw = gzip.decompress(path.read_bytes())
+    except (OSError, gzip.BadGzipFile, EOFError) as exc:
+        logger.warning("could not read existing database %s: %s", path, exc)
+        return []
+
+    entries: List[dict] = []
+    for line in raw.split(b"\n"):
+        if not line:
+            continue
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError as exc:
+            logger.warning("skipping malformed line in %s: %s", path, exc)
+            continue
+        if not isinstance(row, dict) or "string" not in row:
+            continue
+        entries.append(
+            {
+                "string": row.get("string"),
+                "library_name": row.get("library_name"),
+                "library_version": row.get("library_version"),
+                "file_path": row.get("file_path"),
+                "function_name": row.get("function_name"),
+                "line_number": row.get("line_number"),
+            }
+        )
+    return entries
+
+
+def write_entries(entries: List[dict], output_path: pathlib.Path) -> None:
+    """Write entries to a gzip-compressed JSONL file."""
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    with gzip.open(output_path, "wt", encoding="utf-8") as f:
+        for entry in entries:
+            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def merge_entries(
+    new_entries: List[dict],
+    existing_entries: List[dict],
+    deduplicate: bool,
+) -> List[dict]:
+    """Combine new and existing entries, with new taking precedence on conflict.
+
+    When ``deduplicate`` is true the result contains at most one entry per
+    unique string value; otherwise the lists are concatenated as-is.
+    """
+    if not new_entries:
+        return list(existing_entries)
+    if not existing_entries:
+        return list(new_entries)
+    if not deduplicate:
+        return list(existing_entries) + list(new_entries)
+
+    seen: Dict[str, dict] = {}
+    # Iterate new first so that freshly-built entries win on string collisions,
+    # which is the right behavior when the underlying library version changes.
+    for entry in list(new_entries) + list(existing_entries):
+        key = entry["string"]
+        if key not in seen:
+            seen[key] = entry
+    return list(seen.values())
+
+
+def write_library_database(
+    metrics: LibraryMetrics,
+    entries: List[dict],
+    output_dir: pathlib.Path,
+    converter: Converter,
+) -> LibraryMetrics:
+    """Write the per-library JSONL.gz and update metrics. Returns metrics."""
+    output_path = output_dir / f"{metrics.library}.jsonl.gz"
+
+    if not entries:
+        if output_path.exists():
+            output_path.unlink()
+        logger.info("%s: removed empty database %s", metrics.library, output_path)
+        metrics.num_string_entries = 0
+        metrics.num_function_name_entries = 0
+        metrics.num_shared_removed = 0
+        metrics.total_entries = 0
+        return metrics
+
+    counts = converter.write(entries, output_path)
+    metrics.num_string_entries = counts["num_string_entries"]
+    metrics.num_function_name_entries = counts["num_function_name_entries"]
+    metrics.total_entries = counts["total_entries"]
+    logger.info(
+        "%s: wrote %s (%d entries, %d removed as shared with other libraries)",
+        metrics.library,
+        output_path,
+        metrics.total_entries,
+        metrics.num_shared_removed,
+    )
+    return metrics
+
+
+def load_config(path: pathlib.Path) -> dict:
+    """Load build configuration from a JSON file."""
+    data = json.loads(path.read_text())
+    if not isinstance(data, dict):
+        raise ValueError(f"config file {path} must contain a JSON object")
+    return data
+
+
+def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
+    # First pass: figure out if a config file was provided.
+    pre_parser = argparse.ArgumentParser(add_help=False)
+    pre_parser.add_argument("--config", type=pathlib.Path, default=None)
+    pre_args, _ = pre_parser.parse_known_args(argv)
+
+    config: dict = {}
+    if pre_args.config:
+        config = load_config(pre_args.config)
+    elif "CONFIG" in os.environ:
+        config = load_config(pathlib.Path(os.environ["CONFIG"]))
+
+    # Defaults are taken from (lowest to highest precedence):
+    #   built-in constants < config file < environment variables < CLI args
+    defaults = {
+        "triplet": config.get("triplet", os.environ.get("TRIPLET", DEFAULT_TRIPLET)),
+        "compiler": config.get("compiler", os.environ.get("COMPILER", DEFAULT_COMPILER)),
+        "profile": config.get("profile", os.environ.get("PROFILE", DEFAULT_PROFILE)),
+        "libraries": config.get(
+            "libraries",
+            os.environ.get("LIBRARIES", ",".join(DEFAULT_LIBRARIES)).split(","),
+        ),
+    }
+
+    parser = argparse.ArgumentParser(
+        description="Build OSS string databases from vcpkg libraries.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        parents=[pre_parser],
+    )
+    parser.set_defaults(**defaults)
+    parser.add_argument(
+        "--triplet",
+        help="vcpkg triplet",
+    )
+    parser.add_argument(
+        "--compiler",
+        help="compiler label passed to jh",
+    )
+    parser.add_argument(
+        "--profile",
+        help="build profile label passed to jh",
+    )
+    parser.add_argument(
+        "--libraries",
+        nargs="+",
+        help="libraries to build",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=pathlib.Path,
+        default=pathlib.Path(__file__).parent,
+        help="directory for generated .jsonl.gz files and metrics",
+    )
+    parser.add_argument(
+        "--vcpkg-root",
+        type=pathlib.Path,
+        default=os.environ.get("VCPKG_ROOT", None),
+        help="vcpkg installation root",
+    )
+    parser.add_argument(
+        "--jh-path",
+        type=pathlib.Path,
+        default=os.environ.get("JH_PATH", None),
+        help="path to an existing jh binary",
+    )
+    parser.add_argument(
+        "--lancelot-dir",
+        type=pathlib.Path,
+        default=os.environ.get("LANCELOT_DIR", None),
+        help="directory containing lancelot source; jh will be built if --jh-path is not given",
+    )
+    parser.add_argument(
+        "--no-function-names",
+        action="store_true",
+        help="do not emit function-name-as-string entries",
+    )
+    parser.add_argument(
+        "--no-deduplicate",
+        action="store_true",
+        help="emit one JSON object per CSV row instead of one per unique string",
+    )
+    parser.add_argument(
+        "--continue-on-error",
+        action="store_true",
+        help="continue building remaining libraries if one fails and exit successfully",
+    )
+    parser.add_argument(
+        "--log-level",
+        default=os.environ.get("LOG_LEVEL", "INFO"),
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        help="logging level",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = parse_args(argv)
+    logging.getLogger().setLevel(args.log_level.upper())
+
+    config = BuildConfig(
+        triplet=args.triplet,
+        compiler=args.compiler,
+        profile=args.profile,
+        libraries=[lib.strip() for lib in args.libraries if lib.strip()],
+        output_dir=args.output_dir.resolve(),
+        vcpkg_root=args.vcpkg_root.resolve() if args.vcpkg_root else None,
+        jh_path=args.jh_path.resolve() if args.jh_path else None,
+        lancelot_dir=args.lancelot_dir.resolve() if args.lancelot_dir else None,
+        emit_function_names=not args.no_function_names,
+        deduplicate=not args.no_deduplicate,
+        continue_on_error=args.continue_on_error,
+    )
+
+    logger.info("configuration: %s", config)
+
+    vcpkg = Vcpkg(config.vcpkg_root)
+    jh = JHExtractor(config.jh_path, config.lancelot_dir)
+    converter = Converter(
+        emit_function_names=config.emit_function_names,
+        deduplicate=config.deduplicate,
+    )
+
+    config.output_dir.mkdir(parents=True, exist_ok=True)
+
+    metrics: List[LibraryMetrics] = []
+    per_library_new: Dict[str, List[dict]] = {}
+    failed = False
+    for library in config.libraries:
+        metric, entries = build_library(library, config, vcpkg, jh, converter)
+        metrics.append(metric)
+        # Even on error, preserve any partial entries so cross-library dedup
+        # still sees them (and so partial results aren't lost).
+        per_library_new[library] = entries
+        if metric.error:
+            failed = True
+            if not config.continue_on_error:
+                break
+
+    # Discover all existing .jsonl.gz databases in the output directory. These
+    # include libraries we are rebuilding (whose fresh entries will be merged
+    # in) and libraries we are leaving alone (which still participate in
+    # cross-library dedup and may need to be rewritten if shared strings change).
+    existing_files = sorted(config.output_dir.glob("*.jsonl.gz"))
+    metric_by_lib: Dict[str, LibraryMetrics] = {m.library: m for m in metrics}
+
+    def _lib_name_from_path(p: pathlib.Path) -> str:
+        # p.name looks like "<lib>.jsonl.gz"; Path.stem would only strip ".gz".
+        suffix = ".jsonl.gz"
+        if p.name.endswith(suffix):
+            return p.name[: -len(suffix)]
+        return p.stem
+
+    merged: Dict[str, List[dict]] = {}
+    for path in existing_files:
+        lib = _lib_name_from_path(path)
+        existing = load_existing_entries(path)
+        if lib in per_library_new:
+            merged[lib] = merge_entries(per_library_new[lib], existing, config.deduplicate)
+        elif existing:
+            merged[lib] = existing
+
+    # Libraries being built for the first time (no pre-existing file).
+    for lib, new_entries in per_library_new.items():
+        if lib not in merged:
+            merged[lib] = list(new_entries)
+
+    # Cross-library dedup across the full set: rebuilt + preserved.
+    shared_strings = find_shared_strings({lib: entries for lib, entries in merged.items() if entries})
+    if shared_strings:
+        logger.info(
+            "removing %d string(s) shared across 2+ libraries from every database",
+            len(shared_strings),
+        )
+
+    # Write rebuilt libraries and update their metrics.
+    for metric in metrics:
+        entries = merged.get(metric.library, [])
+        if shared_strings and entries:
+            before = len(entries)
+            entries = [e for e in entries if e["string"] not in shared_strings]
+            metric.num_shared_removed = before - len(entries)
+        else:
+            metric.num_shared_removed = 0
+        write_library_database(metric, entries, config.output_dir, converter)
+
+    # Rewrite preserved libraries only if the cross-lib dedup actually removed
+    # something from them. (If we just rebuilt them, write_library_database
+    # already handled it above.)
+    rebuilt_libs = set(metric_by_lib.keys())
+    preserved_metrics: List[dict] = []
+    for lib in sorted(merged.keys()):
+        if lib in rebuilt_libs:
+            continue
+        entries = merged[lib]
+        filtered = [e for e in entries if e["string"] not in shared_strings] if shared_strings else list(entries)
+        if {e["string"] for e in filtered} == {e["string"] for e in entries}:
+            logger.debug("%s: unchanged, leaving %s alone", lib, f"{lib}.jsonl.gz")
+            continue
+        output_path = config.output_dir / f"{lib}.jsonl.gz"
+        if not filtered:
+            output_path.unlink(missing_ok=True)
+            logger.info("%s: removed empty database %s", lib, output_path)
+        else:
+            write_entries(filtered, output_path)
+            logger.info(
+                "%s: rewrote %s (%d entries after cross-lib dedup)",
+                lib,
+                output_path,
+                len(filtered),
+            )
+        preserved_metrics.append(
+            {
+                "library": lib,
+                "version": filtered[0]["library_version"] if filtered else None,
+                "triplet": config.triplet,
+                "num_string_entries": sum(
+                    1
+                    for e in filtered
+                    if e.get("function_name") is not None and e.get("function_name") != e.get("string")
+                ),
+                "num_function_name_entries": sum(
+                    1
+                    for e in filtered
+                    if e.get("function_name") is not None and e.get("function_name") == e.get("string")
+                ),
+                "num_raw_entries": len(entries),
+                "num_shared_removed": len(entries) - len(filtered),
+                "total_entries": len(filtered),
+                "duration_seconds": 0.0,
+                "error": None,
+            }
+        )
+
+    summary = {
+        "triplet": config.triplet,
+        "compiler": config.compiler,
+        "profile": config.profile,
+        "libraries": [m.as_dict() for m in metrics],
+        "preserved_libraries": preserved_metrics,
+        "successful": sum(1 for m in metrics if not m.error),
+        "failed": sum(1 for m in metrics if m.error),
+        "num_shared_strings_removed": len(shared_strings),
+    }
+
+    metrics_path = config.output_dir / "build_metrics.json"
+    metrics_path.write_text(json.dumps(summary, indent=2))
+    logger.info("wrote metrics to %s", metrics_path)
+
+    if failed:
+        logger.error("one or more libraries failed to build")
+        if config.continue_on_error:
+            return 0
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/floss/qs/db/data/oss/readme.md b/floss/qs/db/data/oss/readme.md
index 29f21e80d..1c8acfa79 100644
--- a/floss/qs/db/data/oss/readme.md
+++ b/floss/qs/db/data/oss/readme.md
@@ -94,3 +94,19 @@ These files are then gzip'd:
 ```console
 $  gzip -c zlib.jsonl > zlib.jsonl.gz
 ```
+
+The `build_oss_db.py` script automates the steps above and additionally
+removes strings that appear in two or more libraries from every database.
+A shared string is not a unique identifier for any specific library, so
+attributing it would produce false-positive library matches at query time.
+
+The script also merges into any `.jsonl.gz` already present in the output
+directory rather than replacing them. For each library being rebuilt, the
+newly-extracted entries are combined with the existing entries (new entries
+win on string collisions, which is the right behavior when the library
+version changes). Libraries that are not in the current `--libraries` list
+are also re-checked against the cross-library dedup and rewritten if the
+shared-string set changes. This makes it safe to incrementally add a new
+library (e.g. `python build_oss_db.py --libraries newlib`) without losing
+the other databases, and safe to bump one library's version while leaving
+its previously-committed entries around.

From e79737270647822648445f1f6d83e8ffcd2c791b Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Sat, 4 Jul 2026 20:54:11 +0530
Subject: [PATCH 02/19] ci(oss-db): add bi-weekly rebuild workflow

Adds .github/workflows/build-oss-db.yml that runs the new build script on
a bi-weekly cron (the 1st and 15th of each month at 00:00 UTC) and on
manual dispatch. After each run, the workflow opens a PR against
quantumstrand with the regenerated .jsonl.gz files if any of them changed.

The workflow caches Cargo and vcpkg to keep build times reasonable, builds
jh from the lancelot source, and runs the build script with
--continue-on-error so a single failed library does not block the others.
lancelot is pinned to williballenthin/lancelot HEAD.
---
 .github/workflows/build-oss-db.yml  | 109 ++++++++++++++++++++++++++++
 floss/qs/db/data/oss/libraries.json |  73 +++++++++++++++++++
 2 files changed, 182 insertions(+)
 create mode 100644 .github/workflows/build-oss-db.yml
 create mode 100644 floss/qs/db/data/oss/libraries.json

diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml
new file mode 100644
index 000000000..725e20dde
--- /dev/null
+++ b/.github/workflows/build-oss-db.yml
@@ -0,0 +1,109 @@
+name: Build OSS String Databases
+
+on:
+  # Rebuild the databases bi-weekly (1st and 15th of each month at 00:00 UTC).
+  schedule:
+    - cron: '0 0 1,15 * *'
+
+  # Allow manual runs from the Actions tab.
+  workflow_dispatch:
+
+# Cancel any in-progress run when a newer one starts.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  build-databases:
+    name: Build OSS string databases
+    runs-on: windows-latest
+
+    env:
+      # Workaround for zydis 3.1.3's CMakeLists.txt requiring an older
+      # cmake_minimum_required syntax. Remove once Lancelot updates zydis.
+      CMAKE_POLICY_VERSION_MINIMUM: '3.5'
+
+    steps:
+      - name: Checkout flare-floss
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          path: flare-floss
+
+      - name: Checkout lancelot
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          repository: williballenthin/lancelot
+          path: lancelot
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.12'
+
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+
+      - name: Cache Cargo
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            lancelot/target/
+          key: ${{ runner.os }}-cargo-jh-${{ hashFiles('lancelot/**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-jh-
+
+      - name: Cache vcpkg
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            C:\vcpkg\installed
+            C:\vcpkg\downloads
+          key: ${{ runner.os }}-vcpkg-oss-db-${{ hashFiles('flare-floss/floss/qs/db/data/oss/build_oss_db.py') }}
+          restore-keys: |
+            ${{ runner.os }}-vcpkg-oss-db-
+
+      - name: Build jh
+        run: |
+          cargo build --release -p lancelot-bin
+        working-directory: lancelot
+
+      - name: Build OSS databases
+        run: |
+          python floss/qs/db/data/oss/build_oss_db.py `
+            --config floss\qs\db\data\oss\libraries.json `
+            --jh-path ..\lancelot\target\release\jh.exe `
+            --output-dir floss\qs\db\data\oss `
+            --continue-on-error
+        working-directory: flare-floss
+
+      - name: Show metrics summary
+        if: success() || failure()
+        run: |
+          Get-Content floss\qs\db\data\oss\build_metrics.json
+        working-directory: flare-floss
+
+      - name: Create Pull Request if databases changed
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0
+        with:
+          path: flare-floss
+          base: quantumstrand
+          commit-message: 'chore(oss-db): update string databases'
+          title: 'Update OSS string databases'
+          body: |
+            Automated bi-weekly rebuild of the OSS string databases.
+
+            - Triplet: `x64-windows-static`
+            - Compiler: `msvc143`
+            - Library list: see `floss/qs/db/data/oss/libraries.json`
+
+            Per-library entry counts and timing are in `floss/qs/db/data/oss/build_metrics.json`.
+          branch: update-oss-string-databases
+          delete-branch: true
diff --git a/floss/qs/db/data/oss/libraries.json b/floss/qs/db/data/oss/libraries.json
new file mode 100644
index 000000000..a6aef2064
--- /dev/null
+++ b/floss/qs/db/data/oss/libraries.json
@@ -0,0 +1,73 @@
+{
+  "triplet": "x64-windows-static",
+  "compiler": "msvc143",
+  "profile": "release",
+  "libraries": [
+    "openssl",
+    "capnproto",
+    "raylib",
+    "libgit2",
+    "libxml2",
+    "libimobiledevice",
+    "libarchive",
+    "libmysql",
+    "libhv",
+    "cello",
+    "czmq",
+    "boost-test",
+    "wolfssl",
+    "boost-wave",
+    "curl",
+    "poco",
+    "ncurses",
+    "nng",
+    "clblas",
+    "libsrt",
+    "sail",
+    "mbedtls",
+    "boost-log",
+    "libsndfile",
+    "libevent",
+    "libcoap",
+    "sqlite3",
+    "sdl2",
+    "libbson",
+    "mongoose",
+    "s2n",
+    "duktape",
+    "libpcap",
+    "expat",
+    "libuv",
+    "llhttp",
+    "jansson",
+    "tinyfiledialogs",
+    "hiredis",
+    "liboqs",
+    "zyre",
+    "boost-json",
+    "boost-serialization",
+    "tre",
+    "zlib",
+    "boost-graph",
+    "apr",
+    "libev",
+    "flatcc",
+    "boost-filesystem",
+    "boost-iostreams",
+    "oniguruma",
+    "libyaml",
+    "cunit",
+    "avro-c",
+    "lz4",
+    "parson",
+    "json-c",
+    "libzip",
+    "boost-container",
+    "lzo",
+    "liblzma",
+    "boost-chrono",
+    "lmdb",
+    "binn",
+    "boost-thread"
+  ]
+}

From a6bf8b63fe962546c41f0531c05386f706c87ec5 Mon Sep 17 00:00:00 2001
From: lakshit verma <51952975+vee1e@users.noreply.github.com>
Date: Sat, 4 Jul 2026 21:15:23 +0530
Subject: [PATCH 03/19] Apply suggestions from code review

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 floss/qs/db/data/oss/build_oss_db.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/floss/qs/db/data/oss/build_oss_db.py b/floss/qs/db/data/oss/build_oss_db.py
index 418217589..0fea0a796 100755
--- a/floss/qs/db/data/oss/build_oss_db.py
+++ b/floss/qs/db/data/oss/build_oss_db.py
@@ -241,7 +241,9 @@ def find_package_libs(self, library: str, triplet: str) -> List[pathlib.Path]:
         """Return static-library files (.lib/.a) owned by the given package."""
         # vcpkg records installed files in <root>/installed/vcpkg/info/<package>_<version>_<triplet>.list
         pattern = re.compile(re.escape(library) + r"_.*?_" + re.escape(triplet) + r"\.list$")
-        list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)]
+        list_files = []
+        if self.info_dir.exists():
+            list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)]
 
         if not list_files:
             # Fallback: scan the whole lib directory. This is less precise but works
@@ -886,8 +888,12 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
             len(shared_strings),
         )
 
+    # Write rebuilt libraries and update their metrics.
     # Write rebuilt libraries and update their metrics.
     for metric in metrics:
+        if metric.error:
+            logger.warning("%s: skipping database write due to build error", metric.library)
+            continue
         entries = merged.get(metric.library, [])
         if shared_strings and entries:
             before = len(entries)
@@ -896,7 +902,6 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         else:
             metric.num_shared_removed = 0
         write_library_database(metric, entries, config.output_dir, converter)
-
     # Rewrite preserved libraries only if the cross-lib dedup actually removed
     # something from them. (If we just rebuilt them, write_library_database
     # already handled it above.)

From e1cb1d099d4ea7504698435dcc068616ef2f2abd Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Sun, 5 Jul 2026 02:58:38 +0530
Subject: [PATCH 04/19] fix(script): remove unnecessary defaults

---
 floss/qs/db/data/oss/build_oss_db.py | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/floss/qs/db/data/oss/build_oss_db.py b/floss/qs/db/data/oss/build_oss_db.py
index 0fea0a796..1f4e75441 100755
--- a/floss/qs/db/data/oss/build_oss_db.py
+++ b/floss/qs/db/data/oss/build_oss_db.py
@@ -64,22 +64,6 @@
 logger = logging.getLogger("build_oss_db")
 
 
-# Fallback library list when neither --libraries nor libraries.json is
-# provided. The actual production list lives in libraries.json.
-DEFAULT_LIBRARIES: List[str] = [
-    "openssl",
-    "sqlite3",
-    "curl",
-    "mbedtls",
-    "jemalloc",
-]
-
-# Matches the values documented in readme.md.
-DEFAULT_TRIPLET = "x64-windows-static"
-DEFAULT_COMPILER = "msvc143"
-DEFAULT_PROFILE = "release"
-
-
 class BuildError(Exception):
     """Raised when a single library cannot be built; the caller decides whether to abort or continue."""
 
@@ -729,12 +713,11 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
     # Defaults are taken from (lowest to highest precedence):
     #   built-in constants < config file < environment variables < CLI args
     defaults = {
-        "triplet": config.get("triplet", os.environ.get("TRIPLET", DEFAULT_TRIPLET)),
-        "compiler": config.get("compiler", os.environ.get("COMPILER", DEFAULT_COMPILER)),
-        "profile": config.get("profile", os.environ.get("PROFILE", DEFAULT_PROFILE)),
+        "triplet": config.get("triplet"),
+        "compiler": config.get("compiler"),
+        "profile": config.get("profile"),
         "libraries": config.get(
             "libraries",
-            os.environ.get("LIBRARIES", ",".join(DEFAULT_LIBRARIES)).split(","),
         ),
     }
 

From fb06ac32d8e6c9c110507d384f90408feec38388 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Sun, 5 Jul 2026 18:13:35 +0530
Subject: [PATCH 05/19] fix(script): remove linux only libraries to keep the
 focus on PE

---
 floss/qs/db/data/oss/libraries.json | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/floss/qs/db/data/oss/libraries.json b/floss/qs/db/data/oss/libraries.json
index a6aef2064..e1cdef02f 100644
--- a/floss/qs/db/data/oss/libraries.json
+++ b/floss/qs/db/data/oss/libraries.json
@@ -19,7 +19,6 @@
     "boost-wave",
     "curl",
     "poco",
-    "ncurses",
     "nng",
     "clblas",
     "libsrt",
@@ -33,7 +32,6 @@
     "sdl2",
     "libbson",
     "mongoose",
-    "s2n",
     "duktape",
     "libpcap",
     "expat",
@@ -42,7 +40,6 @@
     "jansson",
     "tinyfiledialogs",
     "hiredis",
-    "liboqs",
     "zyre",
     "boost-json",
     "boost-serialization",

From 4f6044bef4ce36bf77461bf3d3cb6ce252f8dd55 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Sun, 5 Jul 2026 21:14:19 +0530
Subject: [PATCH 06/19] chore(script): move the build db script to the global
 scripts dir

---
 .github/workflows/build-oss-db.yml                | 4 ++--
 {floss/qs/db/data/oss => scripts}/build_oss_db.py | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename {floss/qs/db/data/oss => scripts}/build_oss_db.py (100%)
 mode change 100755 => 100644

diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml
index 725e20dde..63cde69c7 100644
--- a/.github/workflows/build-oss-db.yml
+++ b/.github/workflows/build-oss-db.yml
@@ -66,7 +66,7 @@ jobs:
           path: |
             C:\vcpkg\installed
             C:\vcpkg\downloads
-          key: ${{ runner.os }}-vcpkg-oss-db-${{ hashFiles('flare-floss/floss/qs/db/data/oss/build_oss_db.py') }}
+          key: ${{ runner.os }}-vcpkg-oss-db-${{ hashFiles('flare-floss/scripts/build_oss_db.py') }}
           restore-keys: |
             ${{ runner.os }}-vcpkg-oss-db-
 
@@ -77,7 +77,7 @@ jobs:
 
       - name: Build OSS databases
         run: |
-          python floss/qs/db/data/oss/build_oss_db.py `
+          python scripts/build_oss_db.py `
             --config floss\qs\db\data\oss\libraries.json `
             --jh-path ..\lancelot\target\release\jh.exe `
             --output-dir floss\qs\db\data\oss `
diff --git a/floss/qs/db/data/oss/build_oss_db.py b/scripts/build_oss_db.py
old mode 100755
new mode 100644
similarity index 100%
rename from floss/qs/db/data/oss/build_oss_db.py
rename to scripts/build_oss_db.py

From d9f704b8f937a64281bc4df04d502982911ba04f Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Mon, 6 Jul 2026 08:16:24 +0530
Subject: [PATCH 07/19] docs(script): add copyright

---
 scripts/build_oss_db.py | 39 ++++++++++++++-------------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index 1f4e75441..5c7d39a87 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -1,4 +1,18 @@
 #!/usr/bin/env python3
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """
 Build OSS string databases from vcpkg static libraries.
 
@@ -15,31 +29,6 @@
 are removed from *all* of those libraries. A shared string does not uniquely
 identify a library, so attributing it to any specific one would be a false
 positive at query time.
-
-By default, the script merges newly-extracted entries with any pre-existing
-``.jsonl.gz`` in the output directory rather than replacing them. This makes
-it safe to incrementally add a library (e.g. ``--libraries newlib``) without
-losing the others, and safe to bump one library's version while leaving its
-previously-committed entries around. Libraries that were not rebuilt still
-participate in the cross-library dedup pass and are rewritten if the shared
-string set changes.
-
-Example usage:
-
-  # Default: build the top 5 largest databases using x64-windows-static.
-  python build_oss_db.py --lancelot-dir ~/Projects/Mandiant/lancelot
-
-  # Build a specific library with a different triplet.
-  python build_oss_db.py --triplet x64-mingw-static --libraries openssl
-
-  # Inside a container where vcpkg/jh are already on PATH.
-  python build_oss_db.py --triplet x64-linux
-
-Environment variables (used as defaults when CLI flags are omitted):
-
-  VCPKG_ROOT          root directory of a vcpkg installation
-  LANCELOT_DIR        directory containing the lancelot source (for building jh)
-  JH_PATH             path to an existing jh binary
 """
 
 from __future__ import annotations

From 8ea3fe1f39b907d20c0680e10377141e7738dba1 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 03:00:56 +0530
Subject: [PATCH 08/19] fix(script): remove problematic and unneeded lib
 `clblas`

---
 floss/qs/db/data/oss/libraries.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/floss/qs/db/data/oss/libraries.json b/floss/qs/db/data/oss/libraries.json
index e1cdef02f..32bf0c1f7 100644
--- a/floss/qs/db/data/oss/libraries.json
+++ b/floss/qs/db/data/oss/libraries.json
@@ -20,7 +20,6 @@
     "curl",
     "poco",
     "nng",
-    "clblas",
     "libsrt",
     "sail",
     "mbedtls",

From 0b6f915b13373f12a1e87cbef5e83a6fa589b29d Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 03:02:17 +0530
Subject: [PATCH 09/19] fix(workflow): enable LFS checkout for build-oss-db

The OSS string databases in floss/qs/db/data/oss/*.jsonl.gz are tracked
via Git LFS (see .gitattributes), but the workflow's actions/checkout
step did not pass lfs: true. As a result the LFS pointer text was being
read as the file content and gzip decoding failed with
'Not a gzipped file (b've...)'.
---
 .github/workflows/build-oss-db.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml
index 63cde69c7..3a5cc028c 100644
--- a/.github/workflows/build-oss-db.yml
+++ b/.github/workflows/build-oss-db.yml
@@ -32,12 +32,14 @@ jobs:
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
           path: flare-floss
+          lfs: true
 
       - name: Checkout lancelot
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
           repository: williballenthin/lancelot
           path: lancelot
+          lfs: true
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0

From cf972103f4d4ed49a95ac50bfd1db15faffc5e1c Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 03:03:15 +0530
Subject: [PATCH 10/19] chore(workflow): drop Cargo and vcpkg caching from
 build-oss-db

---
 .github/workflows/build-oss-db.yml | 23 -----------------------
 1 file changed, 23 deletions(-)

diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml
index 3a5cc028c..635ef79e1 100644
--- a/.github/workflows/build-oss-db.yml
+++ b/.github/workflows/build-oss-db.yml
@@ -49,29 +49,6 @@ jobs:
       - name: Set up Rust
         uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
 
-      - name: Cache Cargo
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            lancelot/target/
-          key: ${{ runner.os }}-cargo-jh-${{ hashFiles('lancelot/**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-jh-
-
-      - name: Cache vcpkg
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            C:\vcpkg\installed
-            C:\vcpkg\downloads
-          key: ${{ runner.os }}-vcpkg-oss-db-${{ hashFiles('flare-floss/scripts/build_oss_db.py') }}
-          restore-keys: |
-            ${{ runner.os }}-vcpkg-oss-db-
-
       - name: Build jh
         run: |
           cargo build --release -p lancelot-bin

From f0872a3ffb7307fb549df4720d324b500fbf19a6 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 03:12:04 +0530
Subject: [PATCH 11/19] feat(oss-db): stop deduping shared strings; stamp
 cross-library count

Strings that appeared in 2+ libraries used to be deleted from every
database, which dropped legitimate indicators when a dependency was
vendored (e.g. zlib sources copied into curl).

Instead, every entry is now stamped with the number of distinct
libraries the string appears in (OpenSourceString.count). count=1 means
the string is unique to one library; count>=2 indicates a shared
string that the consumer should weight accordingly. Strings are kept
in every database they appear in, so the vendored-dependency case
preserves the original zlib indicator.

The loader schema gains a defaulted 'count: int = 1' field, so legacy
.jsonl.gz files (no count) still decode and are interpreted as
count=1. Preserved (non-rebuilt) libraries are only rewritten when
their per-string counts actually change, so the workflow does not
produce spurious diffs on runs that don't add or remove libraries.
---
 floss/qs/db/oss.py      |   5 ++
 scripts/build_oss_db.py | 124 +++++++++++++++++++++-------------------
 2 files changed, 71 insertions(+), 58 deletions(-)

diff --git a/floss/qs/db/oss.py b/floss/qs/db/oss.py
index 1d0d4c72d..58b5e1dd0 100644
--- a/floss/qs/db/oss.py
+++ b/floss/qs/db/oss.py
@@ -15,6 +15,11 @@ class OpenSourceString(msgspec.Struct):
     file_path: str | None = None
     function_name: str | None = None
     line_number: int | None = None
+    # Number of distinct libraries the string was observed in across the full
+    # OSS database set. count=1 means the string is unique to this library;
+    # higher counts indicate strings shared across multiple libraries (e.g.
+    # vendored dependencies or common runtime strings).
+    count: int = 1
 
 
 @dataclass
diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index 5c7d39a87..f953b5d3d 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -25,10 +25,14 @@
 It is intentionally modular so the underlying extractor (jh today) can be
 swapped for a more minimal tool later without rewriting the orchestration.
 
-After every library is parsed, strings that appear in two or more libraries
-are removed from *all* of those libraries. A shared string does not uniquely
-identify a library, so attributing it to any specific one would be a false
-positive at query time.
+After every library is parsed, each string's ``count`` is set to the number
+of distinct libraries the string appears in across the full OSS database
+set. The count is exposed via OpenSourceString.count so that consumers can
+distinguish strings unique to one library (count=1) from strings shared
+across many libraries (count>=2). Strings are NOT removed across
+libraries: a string observed in both zlib and curl (e.g. when zlib is
+vendored into curl) stays in both databases with count=2, rather than
+being dropped from both as a "shared" string.
 """
 
 from __future__ import annotations
@@ -86,7 +90,6 @@ class LibraryMetrics:
     num_string_entries: int = 0
     num_function_name_entries: int = 0
     num_raw_entries: int = 0
-    num_shared_removed: int = 0
     total_entries: int = 0
     duration_seconds: float = 0.0
     error: Optional[str] = None
@@ -101,7 +104,6 @@ def as_dict(self) -> dict:
             "num_string_entries": self.num_string_entries,
             "num_function_name_entries": self.num_function_name_entries,
             "num_raw_entries": self.num_raw_entries,
-            "num_shared_removed": self.num_shared_removed,
             "total_entries": self.total_entries,
             "duration_seconds": round(self.duration_seconds, 2),
             "error": self.error,
@@ -554,24 +556,6 @@ def build_library(
     return metrics, entries
 
 
-def find_shared_strings(per_library_entries: Dict[str, List[dict]]) -> Set[str]:
-    """Return the set of strings that appear in two or more libraries.
-
-    A shared string cannot uniquely identify a library, so it is removed from
-    every library's database to avoid false-positive library attributions.
-    """
-    seen_in: Dict[str, str] = {}
-    shared: Set[str] = set()
-    for library, entries in per_library_entries.items():
-        for entry in entries:
-            string = entry["string"]
-            if string in seen_in and seen_in[string] != library:
-                shared.add(string)
-            else:
-                seen_in[string] = library
-    return shared
-
-
 def load_existing_entries(path: pathlib.Path) -> List[dict]:
     """Load entries from an existing OSS database .jsonl.gz.
 
@@ -606,6 +590,7 @@ def load_existing_entries(path: pathlib.Path) -> List[dict]:
                 "file_path": row.get("file_path"),
                 "function_name": row.get("function_name"),
                 "line_number": row.get("line_number"),
+                "count": row.get("count", 1),
             }
         )
     return entries
@@ -619,6 +604,22 @@ def write_entries(entries: List[dict], output_path: pathlib.Path) -> None:
             f.write(json.dumps(entry, ensure_ascii=False) + "\n")
 
 
+def _entries_unchanged(existing: List[dict], new: List[dict]) -> bool:
+    """Return True if ``existing`` and ``new`` have the same string set and per-string count.
+
+    Used to decide whether a preserved library's on-disk database needs to be
+    rewritten: a rewrite is only required when the string set or any
+    per-string count differs.
+    """
+    existing_strings = {e["string"] for e in existing}
+    new_strings = {e["string"] for e in new}
+    if existing_strings != new_strings:
+        return False
+    existing_counts = {e["string"]: e.get("count", 1) for e in existing}
+    new_counts = {e["string"]: e["count"] for e in new}
+    return existing_counts == new_counts
+
+
 def merge_entries(
     new_entries: List[dict],
     existing_entries: List[dict],
@@ -661,7 +662,6 @@ def write_library_database(
         logger.info("%s: removed empty database %s", metrics.library, output_path)
         metrics.num_string_entries = 0
         metrics.num_function_name_entries = 0
-        metrics.num_shared_removed = 0
         metrics.total_entries = 0
         return metrics
 
@@ -670,11 +670,10 @@ def write_library_database(
     metrics.num_function_name_entries = counts["num_function_name_entries"]
     metrics.total_entries = counts["total_entries"]
     logger.info(
-        "%s: wrote %s (%d entries, %d removed as shared with other libraries)",
+        "%s: wrote %s (%d entries)",
         metrics.library,
         output_path,
         metrics.total_entries,
-        metrics.num_shared_removed,
     )
     return metrics
 
@@ -852,71 +851,81 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         if lib not in merged:
             merged[lib] = list(new_entries)
 
-    # Cross-library dedup across the full set: rebuilt + preserved.
-    shared_strings = find_shared_strings({lib: entries for lib, entries in merged.items() if entries})
-    if shared_strings:
-        logger.info(
-            "removing %d string(s) shared across 2+ libraries from every database",
-            len(shared_strings),
-        )
+    # Count how many distinct libraries each string appears in. A string's
+    # count is the "found in N libraries" signal exposed to the user via
+    # OpenSourceString.count, so that strings unique to one library (count=1)
+    # can be distinguished from strings shared across many libraries
+    # (count>=2). Strings are NOT removed across libraries: when the same
+    # string is observed in both zlib and curl (e.g. zlib is vendored into
+    # curl), it stays in both databases with count=2.
+    count_by_string: Dict[str, int] = {}
+    for entries in merged.values():
+        seen_in_lib: Set[str] = set()
+        for entry in entries:
+            s = entry["string"]
+            if s in seen_in_lib:
+                continue
+            seen_in_lib.add(s)
+            count_by_string[s] = count_by_string.get(s, 0) + 1
+
+    # Stamp the count on every entry that we will write so downstream
+    # consumers can see how many libraries each string appears in.
+    for entries in merged.values():
+        for entry in entries:
+            entry["count"] = count_by_string[entry["string"]]
 
-    # Write rebuilt libraries and update their metrics.
     # Write rebuilt libraries and update their metrics.
     for metric in metrics:
         if metric.error:
             logger.warning("%s: skipping database write due to build error", metric.library)
             continue
         entries = merged.get(metric.library, [])
-        if shared_strings and entries:
-            before = len(entries)
-            entries = [e for e in entries if e["string"] not in shared_strings]
-            metric.num_shared_removed = before - len(entries)
-        else:
-            metric.num_shared_removed = 0
         write_library_database(metric, entries, config.output_dir, converter)
-    # Rewrite preserved libraries only if the cross-lib dedup actually removed
-    # something from them. (If we just rebuilt them, write_library_database
-    # already handled it above.)
+
+    # Rewrite preserved libraries only if their string set or per-string
+    # count changed. (If we just rebuilt them, write_library_database
+    # already handled it above.) The count changes whenever a string is
+    # added to or removed from any library in the merged set, including
+    # new libraries joining the build.
     rebuilt_libs = set(metric_by_lib.keys())
     preserved_metrics: List[dict] = []
     for lib in sorted(merged.keys()):
         if lib in rebuilt_libs:
             continue
         entries = merged[lib]
-        filtered = [e for e in entries if e["string"] not in shared_strings] if shared_strings else list(entries)
-        if {e["string"] for e in filtered} == {e["string"] for e in entries}:
-            logger.debug("%s: unchanged, leaving %s alone", lib, f"{lib}.jsonl.gz")
-            continue
         output_path = config.output_dir / f"{lib}.jsonl.gz"
-        if not filtered:
+        existing = load_existing_entries(output_path)
+        if existing and _entries_unchanged(existing, entries):
+            logger.debug("%s: unchanged, leaving %s alone", lib, output_path.name)
+            continue
+        if not entries:
             output_path.unlink(missing_ok=True)
             logger.info("%s: removed empty database %s", lib, output_path)
         else:
-            write_entries(filtered, output_path)
+            write_entries(entries, output_path)
             logger.info(
-                "%s: rewrote %s (%d entries after cross-lib dedup)",
+                "%s: rewrote %s (%d entries)",
                 lib,
                 output_path,
-                len(filtered),
+                len(entries),
             )
         preserved_metrics.append(
             {
                 "library": lib,
-                "version": filtered[0]["library_version"] if filtered else None,
+                "version": entries[0]["library_version"] if entries else None,
                 "triplet": config.triplet,
                 "num_string_entries": sum(
                     1
-                    for e in filtered
+                    for e in entries
                     if e.get("function_name") is not None and e.get("function_name") != e.get("string")
                 ),
                 "num_function_name_entries": sum(
                     1
-                    for e in filtered
+                    for e in entries
                     if e.get("function_name") is not None and e.get("function_name") == e.get("string")
                 ),
                 "num_raw_entries": len(entries),
-                "num_shared_removed": len(entries) - len(filtered),
-                "total_entries": len(filtered),
+                "total_entries": len(entries),
                 "duration_seconds": 0.0,
                 "error": None,
             }
@@ -930,7 +939,6 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         "preserved_libraries": preserved_metrics,
         "successful": sum(1 for m in metrics if not m.error),
         "failed": sum(1 for m in metrics if m.error),
-        "num_shared_strings_removed": len(shared_strings),
     }
 
     metrics_path = config.output_dir / "build_metrics.json"

From 594ce4a8472e584a76aa6df93b3b050c8e8e7ab6 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 03:13:14 +0530
Subject: [PATCH 12/19] test: rename test_oss_db.py to test_qs_oss_db.py

so that 'pytest -k qs' includes it alongside the other qs tests.
---
 tests/{test_oss_db.py => test_qs_oss_db.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/{test_oss_db.py => test_qs_oss_db.py} (100%)

diff --git a/tests/test_oss_db.py b/tests/test_qs_oss_db.py
similarity index 100%
rename from tests/test_oss_db.py
rename to tests/test_qs_oss_db.py

From 1c1dbfaef29b6d03043423213ef9c6d8c9f29268 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 15:46:24 +0530
Subject: [PATCH 13/19] Revert "feat(oss-db): stop deduping shared strings;
 stamp cross-library count"

This reverts commit f0872a3ffb7307fb549df4720d324b500fbf19a6.
---
 floss/qs/db/oss.py      |   5 --
 scripts/build_oss_db.py | 124 +++++++++++++++++++---------------------
 2 files changed, 58 insertions(+), 71 deletions(-)

diff --git a/floss/qs/db/oss.py b/floss/qs/db/oss.py
index 58b5e1dd0..1d0d4c72d 100644
--- a/floss/qs/db/oss.py
+++ b/floss/qs/db/oss.py
@@ -15,11 +15,6 @@ class OpenSourceString(msgspec.Struct):
     file_path: str | None = None
     function_name: str | None = None
     line_number: int | None = None
-    # Number of distinct libraries the string was observed in across the full
-    # OSS database set. count=1 means the string is unique to this library;
-    # higher counts indicate strings shared across multiple libraries (e.g.
-    # vendored dependencies or common runtime strings).
-    count: int = 1
 
 
 @dataclass
diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index f953b5d3d..5c7d39a87 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -25,14 +25,10 @@
 It is intentionally modular so the underlying extractor (jh today) can be
 swapped for a more minimal tool later without rewriting the orchestration.
 
-After every library is parsed, each string's ``count`` is set to the number
-of distinct libraries the string appears in across the full OSS database
-set. The count is exposed via OpenSourceString.count so that consumers can
-distinguish strings unique to one library (count=1) from strings shared
-across many libraries (count>=2). Strings are NOT removed across
-libraries: a string observed in both zlib and curl (e.g. when zlib is
-vendored into curl) stays in both databases with count=2, rather than
-being dropped from both as a "shared" string.
+After every library is parsed, strings that appear in two or more libraries
+are removed from *all* of those libraries. A shared string does not uniquely
+identify a library, so attributing it to any specific one would be a false
+positive at query time.
 """
 
 from __future__ import annotations
@@ -90,6 +86,7 @@ class LibraryMetrics:
     num_string_entries: int = 0
     num_function_name_entries: int = 0
     num_raw_entries: int = 0
+    num_shared_removed: int = 0
     total_entries: int = 0
     duration_seconds: float = 0.0
     error: Optional[str] = None
@@ -104,6 +101,7 @@ def as_dict(self) -> dict:
             "num_string_entries": self.num_string_entries,
             "num_function_name_entries": self.num_function_name_entries,
             "num_raw_entries": self.num_raw_entries,
+            "num_shared_removed": self.num_shared_removed,
             "total_entries": self.total_entries,
             "duration_seconds": round(self.duration_seconds, 2),
             "error": self.error,
@@ -556,6 +554,24 @@ def build_library(
     return metrics, entries
 
 
+def find_shared_strings(per_library_entries: Dict[str, List[dict]]) -> Set[str]:
+    """Return the set of strings that appear in two or more libraries.
+
+    A shared string cannot uniquely identify a library, so it is removed from
+    every library's database to avoid false-positive library attributions.
+    """
+    seen_in: Dict[str, str] = {}
+    shared: Set[str] = set()
+    for library, entries in per_library_entries.items():
+        for entry in entries:
+            string = entry["string"]
+            if string in seen_in and seen_in[string] != library:
+                shared.add(string)
+            else:
+                seen_in[string] = library
+    return shared
+
+
 def load_existing_entries(path: pathlib.Path) -> List[dict]:
     """Load entries from an existing OSS database .jsonl.gz.
 
@@ -590,7 +606,6 @@ def load_existing_entries(path: pathlib.Path) -> List[dict]:
                 "file_path": row.get("file_path"),
                 "function_name": row.get("function_name"),
                 "line_number": row.get("line_number"),
-                "count": row.get("count", 1),
             }
         )
     return entries
@@ -604,22 +619,6 @@ def write_entries(entries: List[dict], output_path: pathlib.Path) -> None:
             f.write(json.dumps(entry, ensure_ascii=False) + "\n")
 
 
-def _entries_unchanged(existing: List[dict], new: List[dict]) -> bool:
-    """Return True if ``existing`` and ``new`` have the same string set and per-string count.
-
-    Used to decide whether a preserved library's on-disk database needs to be
-    rewritten: a rewrite is only required when the string set or any
-    per-string count differs.
-    """
-    existing_strings = {e["string"] for e in existing}
-    new_strings = {e["string"] for e in new}
-    if existing_strings != new_strings:
-        return False
-    existing_counts = {e["string"]: e.get("count", 1) for e in existing}
-    new_counts = {e["string"]: e["count"] for e in new}
-    return existing_counts == new_counts
-
-
 def merge_entries(
     new_entries: List[dict],
     existing_entries: List[dict],
@@ -662,6 +661,7 @@ def write_library_database(
         logger.info("%s: removed empty database %s", metrics.library, output_path)
         metrics.num_string_entries = 0
         metrics.num_function_name_entries = 0
+        metrics.num_shared_removed = 0
         metrics.total_entries = 0
         return metrics
 
@@ -670,10 +670,11 @@ def write_library_database(
     metrics.num_function_name_entries = counts["num_function_name_entries"]
     metrics.total_entries = counts["total_entries"]
     logger.info(
-        "%s: wrote %s (%d entries)",
+        "%s: wrote %s (%d entries, %d removed as shared with other libraries)",
         metrics.library,
         output_path,
         metrics.total_entries,
+        metrics.num_shared_removed,
     )
     return metrics
 
@@ -851,81 +852,71 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         if lib not in merged:
             merged[lib] = list(new_entries)
 
-    # Count how many distinct libraries each string appears in. A string's
-    # count is the "found in N libraries" signal exposed to the user via
-    # OpenSourceString.count, so that strings unique to one library (count=1)
-    # can be distinguished from strings shared across many libraries
-    # (count>=2). Strings are NOT removed across libraries: when the same
-    # string is observed in both zlib and curl (e.g. zlib is vendored into
-    # curl), it stays in both databases with count=2.
-    count_by_string: Dict[str, int] = {}
-    for entries in merged.values():
-        seen_in_lib: Set[str] = set()
-        for entry in entries:
-            s = entry["string"]
-            if s in seen_in_lib:
-                continue
-            seen_in_lib.add(s)
-            count_by_string[s] = count_by_string.get(s, 0) + 1
-
-    # Stamp the count on every entry that we will write so downstream
-    # consumers can see how many libraries each string appears in.
-    for entries in merged.values():
-        for entry in entries:
-            entry["count"] = count_by_string[entry["string"]]
+    # Cross-library dedup across the full set: rebuilt + preserved.
+    shared_strings = find_shared_strings({lib: entries for lib, entries in merged.items() if entries})
+    if shared_strings:
+        logger.info(
+            "removing %d string(s) shared across 2+ libraries from every database",
+            len(shared_strings),
+        )
 
+    # Write rebuilt libraries and update their metrics.
     # Write rebuilt libraries and update their metrics.
     for metric in metrics:
         if metric.error:
             logger.warning("%s: skipping database write due to build error", metric.library)
             continue
         entries = merged.get(metric.library, [])
+        if shared_strings and entries:
+            before = len(entries)
+            entries = [e for e in entries if e["string"] not in shared_strings]
+            metric.num_shared_removed = before - len(entries)
+        else:
+            metric.num_shared_removed = 0
         write_library_database(metric, entries, config.output_dir, converter)
-
-    # Rewrite preserved libraries only if their string set or per-string
-    # count changed. (If we just rebuilt them, write_library_database
-    # already handled it above.) The count changes whenever a string is
-    # added to or removed from any library in the merged set, including
-    # new libraries joining the build.
+    # Rewrite preserved libraries only if the cross-lib dedup actually removed
+    # something from them. (If we just rebuilt them, write_library_database
+    # already handled it above.)
     rebuilt_libs = set(metric_by_lib.keys())
     preserved_metrics: List[dict] = []
     for lib in sorted(merged.keys()):
         if lib in rebuilt_libs:
             continue
         entries = merged[lib]
-        output_path = config.output_dir / f"{lib}.jsonl.gz"
-        existing = load_existing_entries(output_path)
-        if existing and _entries_unchanged(existing, entries):
-            logger.debug("%s: unchanged, leaving %s alone", lib, output_path.name)
+        filtered = [e for e in entries if e["string"] not in shared_strings] if shared_strings else list(entries)
+        if {e["string"] for e in filtered} == {e["string"] for e in entries}:
+            logger.debug("%s: unchanged, leaving %s alone", lib, f"{lib}.jsonl.gz")
             continue
-        if not entries:
+        output_path = config.output_dir / f"{lib}.jsonl.gz"
+        if not filtered:
             output_path.unlink(missing_ok=True)
             logger.info("%s: removed empty database %s", lib, output_path)
         else:
-            write_entries(entries, output_path)
+            write_entries(filtered, output_path)
             logger.info(
-                "%s: rewrote %s (%d entries)",
+                "%s: rewrote %s (%d entries after cross-lib dedup)",
                 lib,
                 output_path,
-                len(entries),
+                len(filtered),
             )
         preserved_metrics.append(
             {
                 "library": lib,
-                "version": entries[0]["library_version"] if entries else None,
+                "version": filtered[0]["library_version"] if filtered else None,
                 "triplet": config.triplet,
                 "num_string_entries": sum(
                     1
-                    for e in entries
+                    for e in filtered
                     if e.get("function_name") is not None and e.get("function_name") != e.get("string")
                 ),
                 "num_function_name_entries": sum(
                     1
-                    for e in entries
+                    for e in filtered
                     if e.get("function_name") is not None and e.get("function_name") == e.get("string")
                 ),
                 "num_raw_entries": len(entries),
-                "total_entries": len(entries),
+                "num_shared_removed": len(entries) - len(filtered),
+                "total_entries": len(filtered),
                 "duration_seconds": 0.0,
                 "error": None,
             }
@@ -939,6 +930,7 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         "preserved_libraries": preserved_metrics,
         "successful": sum(1 for m in metrics if not m.error),
         "failed": sum(1 for m in metrics if m.error),
+        "num_shared_strings_removed": len(shared_strings),
     }
 
     metrics_path = config.output_dir / "build_metrics.json"

From d272c2ca3216f25b4153ea9bba22c3f7cd50ef61 Mon Sep 17 00:00:00 2001
From: vee1e <vermalucky2004@gmail.com>
Date: Tue, 7 Jul 2026 15:50:09 +0530
Subject: [PATCH 14/19] feat(oss-db): keep strings that appear in multiple
 libraries

Strings that appear in 2+ libraries used to be dropped from every
database they showed up in. This silently removed legitimate
indicators when a dependency was vendored (e.g. zlib sources copied
into curl), because the vendored copy's strings would no longer point
back to zlib.

Now the same string can appear in any number of .jsonl.gz files. The
query tagger in floss/qs/main.py already emits one '#<library>' tag
per matching database, so the consumer can see the overlap directly
(a string tagged '#zlib #curl' is observed in both libraries) and
weight the indicator accordingly.

Removes:
  - find_shared_strings() and the cross-library filter loops in main()
  - the preserved_libraries rewrite block (only existed to handle
    entries removed by cross-lib dedup)
  - num_shared_removed / num_shared_strings_removed from metrics

Within-library dedup is unchanged: the same string still collapses
to one entry per library (unless --no-deduplicate is passed).
---
 scripts/build_oss_db.py | 108 +++++-----------------------------------
 1 file changed, 13 insertions(+), 95 deletions(-)

diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index 5c7d39a87..f2164a0aa 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -25,10 +25,12 @@
 It is intentionally modular so the underlying extractor (jh today) can be
 swapped for a more minimal tool later without rewriting the orchestration.
 
-After every library is parsed, strings that appear in two or more libraries
-are removed from *all* of those libraries. A shared string does not uniquely
-identify a library, so attributing it to any specific one would be a false
-positive at query time.
+Strings are NOT deduped across libraries: a string observed in both zlib
+and curl (e.g. when zlib is vendored into curl) stays in both databases.
+The query tagger already emits one ``#<library>`` tag per matching
+database, so the consumer can see the overlap directly. Within a single
+library, the same string still collapses to one entry (when
+``--no-deduplicate`` is not passed).
 """
 
 from __future__ import annotations
@@ -86,7 +88,6 @@ class LibraryMetrics:
     num_string_entries: int = 0
     num_function_name_entries: int = 0
     num_raw_entries: int = 0
-    num_shared_removed: int = 0
     total_entries: int = 0
     duration_seconds: float = 0.0
     error: Optional[str] = None
@@ -101,7 +102,6 @@ def as_dict(self) -> dict:
             "num_string_entries": self.num_string_entries,
             "num_function_name_entries": self.num_function_name_entries,
             "num_raw_entries": self.num_raw_entries,
-            "num_shared_removed": self.num_shared_removed,
             "total_entries": self.total_entries,
             "duration_seconds": round(self.duration_seconds, 2),
             "error": self.error,
@@ -554,24 +554,6 @@ def build_library(
     return metrics, entries
 
 
-def find_shared_strings(per_library_entries: Dict[str, List[dict]]) -> Set[str]:
-    """Return the set of strings that appear in two or more libraries.
-
-    A shared string cannot uniquely identify a library, so it is removed from
-    every library's database to avoid false-positive library attributions.
-    """
-    seen_in: Dict[str, str] = {}
-    shared: Set[str] = set()
-    for library, entries in per_library_entries.items():
-        for entry in entries:
-            string = entry["string"]
-            if string in seen_in and seen_in[string] != library:
-                shared.add(string)
-            else:
-                seen_in[string] = library
-    return shared
-
-
 def load_existing_entries(path: pathlib.Path) -> List[dict]:
     """Load entries from an existing OSS database .jsonl.gz.
 
@@ -661,7 +643,6 @@ def write_library_database(
         logger.info("%s: removed empty database %s", metrics.library, output_path)
         metrics.num_string_entries = 0
         metrics.num_function_name_entries = 0
-        metrics.num_shared_removed = 0
         metrics.total_entries = 0
         return metrics
 
@@ -670,11 +651,10 @@ def write_library_database(
     metrics.num_function_name_entries = counts["num_function_name_entries"]
     metrics.total_entries = counts["total_entries"]
     logger.info(
-        "%s: wrote %s (%d entries, %d removed as shared with other libraries)",
+        "%s: wrote %s (%d entries)",
         metrics.library,
         output_path,
         metrics.total_entries,
-        metrics.num_shared_removed,
     )
     return metrics
 
@@ -816,8 +796,7 @@ def main(argv: Optional[List[str]] = None) -> int:
     for library in config.libraries:
         metric, entries = build_library(library, config, vcpkg, jh, converter)
         metrics.append(metric)
-        # Even on error, preserve any partial entries so cross-library dedup
-        # still sees them (and so partial results aren't lost).
+        # Even on error, preserve any partial entries so they aren't lost.
         per_library_new[library] = entries
         if metric.error:
             failed = True
@@ -826,8 +805,11 @@ def main(argv: Optional[List[str]] = None) -> int:
 
     # Discover all existing .jsonl.gz databases in the output directory. These
     # include libraries we are rebuilding (whose fresh entries will be merged
-    # in) and libraries we are leaving alone (which still participate in
-    # cross-library dedup and may need to be rewritten if shared strings change).
+    # in) and libraries we are leaving alone. Strings are NOT deduped across
+    # libraries: a string that appears in both zlib and curl (e.g. when zlib
+    # is vendored into curl) stays in both databases. The query tagger already
+    # emits one #library tag per matching database, so the consumer can see
+    # the overlap directly.
     existing_files = sorted(config.output_dir.glob("*.jsonl.gz"))
     metric_by_lib: Dict[str, LibraryMetrics] = {m.library: m for m in metrics}
 
@@ -852,85 +834,21 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
         if lib not in merged:
             merged[lib] = list(new_entries)
 
-    # Cross-library dedup across the full set: rebuilt + preserved.
-    shared_strings = find_shared_strings({lib: entries for lib, entries in merged.items() if entries})
-    if shared_strings:
-        logger.info(
-            "removing %d string(s) shared across 2+ libraries from every database",
-            len(shared_strings),
-        )
-
-    # Write rebuilt libraries and update their metrics.
     # Write rebuilt libraries and update their metrics.
     for metric in metrics:
         if metric.error:
             logger.warning("%s: skipping database write due to build error", metric.library)
             continue
         entries = merged.get(metric.library, [])
-        if shared_strings and entries:
-            before = len(entries)
-            entries = [e for e in entries if e["string"] not in shared_strings]
-            metric.num_shared_removed = before - len(entries)
-        else:
-            metric.num_shared_removed = 0
         write_library_database(metric, entries, config.output_dir, converter)
-    # Rewrite preserved libraries only if the cross-lib dedup actually removed
-    # something from them. (If we just rebuilt them, write_library_database
-    # already handled it above.)
-    rebuilt_libs = set(metric_by_lib.keys())
-    preserved_metrics: List[dict] = []
-    for lib in sorted(merged.keys()):
-        if lib in rebuilt_libs:
-            continue
-        entries = merged[lib]
-        filtered = [e for e in entries if e["string"] not in shared_strings] if shared_strings else list(entries)
-        if {e["string"] for e in filtered} == {e["string"] for e in entries}:
-            logger.debug("%s: unchanged, leaving %s alone", lib, f"{lib}.jsonl.gz")
-            continue
-        output_path = config.output_dir / f"{lib}.jsonl.gz"
-        if not filtered:
-            output_path.unlink(missing_ok=True)
-            logger.info("%s: removed empty database %s", lib, output_path)
-        else:
-            write_entries(filtered, output_path)
-            logger.info(
-                "%s: rewrote %s (%d entries after cross-lib dedup)",
-                lib,
-                output_path,
-                len(filtered),
-            )
-        preserved_metrics.append(
-            {
-                "library": lib,
-                "version": filtered[0]["library_version"] if filtered else None,
-                "triplet": config.triplet,
-                "num_string_entries": sum(
-                    1
-                    for e in filtered
-                    if e.get("function_name") is not None and e.get("function_name") != e.get("string")
-                ),
-                "num_function_name_entries": sum(
-                    1
-                    for e in filtered
-                    if e.get("function_name") is not None and e.get("function_name") == e.get("string")
-                ),
-                "num_raw_entries": len(entries),
-                "num_shared_removed": len(entries) - len(filtered),
-                "total_entries": len(filtered),
-                "duration_seconds": 0.0,
-                "error": None,
-            }
-        )
 
     summary = {
         "triplet": config.triplet,
         "compiler": config.compiler,
         "profile": config.profile,
         "libraries": [m.as_dict() for m in metrics],
-        "preserved_libraries": preserved_metrics,
         "successful": sum(1 for m in metrics if not m.error),
         "failed": sum(1 for m in metrics if m.error),
-        "num_shared_strings_removed": len(shared_strings),
     }
 
     metrics_path = config.output_dir / "build_metrics.json"

From daaf96d6a3951e90a2643d425dc6c6d6ca79bddc Mon Sep 17 00:00:00 2001
From: lakshit verma <vermalucky2004@gmail.com>
Date: Wed, 8 Jul 2026 05:36:30 +0530
Subject: [PATCH 15/19] docs(oss-db): fix outdated readme and stale docstrings
 from code review

Addresses part of #1310.

- floss/qs/db/data/oss/readme.md: rewrite the build_oss_db.py paragraph
  to reflect the current behavior (no cross-library dedup; non-rebuilt
  libraries are not rewritten), and drop trailing whitespace.
- scripts/build_oss_db.py: JHExtractor.extract returns JSONL, not CSV;
  update the docstring and the --no-deduplicate help text accordingly.
- floss/qs/db/data/oss/libraries.json: sort the library list A-Z for
  stable, predictable ordering.
---
 floss/qs/db/data/oss/libraries.json | 104 ++++++++++++++--------------
 floss/qs/db/data/oss/readme.md      |  28 +++-----
 scripts/build_oss_db.py             |   4 +-
 3 files changed, 64 insertions(+), 72 deletions(-)

diff --git a/floss/qs/db/data/oss/libraries.json b/floss/qs/db/data/oss/libraries.json
index 32bf0c1f7..3fc696be1 100644
--- a/floss/qs/db/data/oss/libraries.json
+++ b/floss/qs/db/data/oss/libraries.json
@@ -3,67 +3,67 @@
   "compiler": "msvc143",
   "profile": "release",
   "libraries": [
-    "openssl",
-    "capnproto",
-    "raylib",
-    "libgit2",
-    "libxml2",
-    "libimobiledevice",
-    "libarchive",
-    "libmysql",
-    "libhv",
-    "cello",
-    "czmq",
+    "apr",
+    "avro-c",
+    "binn",
+    "boost-chrono",
+    "boost-container",
+    "boost-filesystem",
+    "boost-graph",
+    "boost-iostreams",
+    "boost-json",
+    "boost-log",
+    "boost-serialization",
     "boost-test",
-    "wolfssl",
+    "boost-thread",
     "boost-wave",
+    "capnproto",
+    "cello",
+    "cunit",
     "curl",
-    "poco",
-    "nng",
-    "libsrt",
-    "sail",
-    "mbedtls",
-    "boost-log",
-    "libsndfile",
-    "libevent",
-    "libcoap",
-    "sqlite3",
-    "sdl2",
-    "libbson",
-    "mongoose",
+    "czmq",
     "duktape",
-    "libpcap",
     "expat",
-    "libuv",
-    "llhttp",
-    "jansson",
-    "tinyfiledialogs",
+    "flatcc",
     "hiredis",
-    "zyre",
-    "boost-json",
-    "boost-serialization",
-    "tre",
-    "zlib",
-    "boost-graph",
-    "apr",
+    "jansson",
+    "json-c",
+    "libarchive",
+    "libbson",
+    "libcoap",
     "libev",
-    "flatcc",
-    "boost-filesystem",
-    "boost-iostreams",
-    "oniguruma",
+    "libevent",
+    "libgit2",
+    "libhv",
+    "libimobiledevice",
+    "liblzma",
+    "libmysql",
+    "libpcap",
+    "libsndfile",
+    "libsrt",
+    "libuv",
+    "libxml2",
     "libyaml",
-    "cunit",
-    "avro-c",
-    "lz4",
-    "parson",
-    "json-c",
     "libzip",
-    "boost-container",
-    "lzo",
-    "liblzma",
-    "boost-chrono",
+    "llhttp",
     "lmdb",
-    "binn",
-    "boost-thread"
+    "lz4",
+    "lzo",
+    "mbedtls",
+    "mongoose",
+    "nng",
+    "oniguruma",
+    "openssl",
+    "parson",
+    "poco",
+    "raylib",
+    "sail",
+    "sdl2",
+    "sqlite3",
+    "tinyfiledialogs",
+    "tre",
+    "wolfssl",
+    "zlib",
+    "zyre"
   ]
 }
diff --git a/floss/qs/db/data/oss/readme.md b/floss/qs/db/data/oss/readme.md
index 1c8acfa79..32e58a5e7 100644
--- a/floss/qs/db/data/oss/readme.md
+++ b/floss/qs/db/data/oss/readme.md
@@ -55,12 +55,12 @@ PS > C:\vcpkg\vcpkg.exe install --triplet x64-windows-static zlib
 
 [jh](https://github.com/williballenthin/lancelot/blob/master/bin/src/bin/jh.rs)
 is a lancelot-based utility that parses AR archives containing COFF object files,
-reconstructs their control flow, finds functions, and extracts features. 
+reconstructs their control flow, finds functions, and extracts features.
 jh extracts numbers, API calls, and strings; we are only interested in the string features.
 
-For each feature, jh emits a CSV line with the fields 
+For each feature, jh emits a CSV line with the fields
   - target triplet
-  - compiler 
+  - compiler
   - library
   - version
   - build profile
@@ -95,18 +95,10 @@ These files are then gzip'd:
 $  gzip -c zlib.jsonl > zlib.jsonl.gz
 ```
 
-The `build_oss_db.py` script automates the steps above and additionally
-removes strings that appear in two or more libraries from every database.
-A shared string is not a unique identifier for any specific library, so
-attributing it would produce false-positive library matches at query time.
-
-The script also merges into any `.jsonl.gz` already present in the output
-directory rather than replacing them. For each library being rebuilt, the
-newly-extracted entries are combined with the existing entries (new entries
-win on string collisions, which is the right behavior when the library
-version changes). Libraries that are not in the current `--libraries` list
-are also re-checked against the cross-library dedup and rewritten if the
-shared-string set changes. This makes it safe to incrementally add a new
-library (e.g. `python build_oss_db.py --libraries newlib`) without losing
-the other databases, and safe to bump one library's version while leaving
-its previously-committed entries around.
+The `build_oss_db.py` script automates the steps above and merges into
+any `.jsonl.gz` already present in the output directory rather than
+replacing them. For each library being rebuilt, the newly-extracted
+entries are combined with the existing entries (new entries win on
+string collisions, which is the right behavior when the library version
+changes).
+
diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index f2164a0aa..22f01f272 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -310,7 +310,7 @@ def extract(
         compiler: str,
         profile: str,
     ) -> str:
-        """Run jh on a single static library and return its CSV output."""
+        """Run jh on a single static library and return its JSONL output."""
         cmd = [
             str(self.jh_path),
             triplet,
@@ -745,7 +745,7 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
     parser.add_argument(
         "--no-deduplicate",
         action="store_true",
-        help="emit one JSON object per CSV row instead of one per unique string",
+        help="emit one JSON object per JSONL row instead of one per unique string",
     )
     parser.add_argument(
         "--continue-on-error",

From 582999e2f9581ee7b2d562222ce01a55477449ed Mon Sep 17 00:00:00 2001
From: lakshit verma <vermalucky2004@gmail.com>
Date: Wed, 8 Jul 2026 05:36:30 +0530
Subject: [PATCH 16/19] refactor(oss-db): address code review; add tests for
 build script

- Remove dead code: Converter.convert and the standalone count_jsonl_rows
  helper are unused anywhere in the repo.
- Avoid double-parsing: fold object/function counting into Converter.parse
  and return a ParseResult(entries, num_objects, num_functions). The old
  code called count_jsonl_rows() and parse() back-to-back, each doing a
  full json.loads pass over the same text.
- Consolidate entry schema: introduce make_db_entry() and route the four
  call sites (3 in parse, 1 in load_existing_entries) through it so the
  six-key dict shape is defined exactly once.
- Add tests/test_build_oss_db.py covering make_db_entry, Converter.parse
  (counts, dedup behavior, emit_function_names, explicit function_name
  rows, malformed lines, empty lines), and load_existing_entries
  (round-trip, missing file, malformed lines, schema tolerance, empty
  file). 22 tests, all passing.

Addresses part of #1310.
---
 scripts/build_oss_db.py    | 136 ++++++--------
 tests/test_build_oss_db.py | 359 +++++++++++++++++++++++++++++++++++++
 2 files changed, 416 insertions(+), 79 deletions(-)
 create mode 100644 tests/test_build_oss_db.py

diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index 22f01f272..ae8164924 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -78,6 +78,34 @@ class BuildConfig:
     continue_on_error: bool = False
 
 
+def make_db_entry(
+    string: str,
+    library_name: str,
+    library_version: str,
+    file_path: Optional[str],
+    function_name: Optional[str],
+    line_number: Optional[int] = None,
+) -> dict:
+    """Construct a database entry using the standard OSS schema."""
+    return {
+        "string": string,
+        "library_name": library_name,
+        "library_version": library_version,
+        "file_path": file_path,
+        "function_name": function_name,
+        "line_number": line_number,
+    }
+
+
+@dataclass
+class ParseResult:
+    """Result of parsing jh JSONL output for a single library."""
+
+    entries: List[dict]
+    num_objects: int
+    num_functions: int
+
+
 @dataclass
 class LibraryMetrics:
     library: str
@@ -346,9 +374,15 @@ def parse(
         jh_text: str,
         library: str,
         version: str,
-    ) -> List[dict]:
-        """Parse jh JSONL into a list of entries (within-library dedup applied if enabled)."""
+    ) -> ParseResult:
+        """Parse jh JSONL into entries and tally object/function counts in one pass.
+
+        Within-library dedup is applied to entries if enabled. Object and
+        function counts reflect the raw (pre-dedup) input, matching the
+        behavior of the previous standalone counter.
+        """
         entries: List[dict] = []
+        objects: Set[str] = set()
         function_names: Set[str] = set()
         explicit_function_names: Set[str] = set()
 
@@ -370,30 +404,17 @@ def parse(
             if not function_name:
                 continue
 
+            objects.add(file_path)
             function_names.add(function_name)
 
             if feat_type == "string":
                 entries.append(
-                    {
-                        "string": value,
-                        "library_name": library,
-                        "library_version": version,
-                        "file_path": file_path,
-                        "function_name": function_name,
-                        "line_number": None,
-                    }
+                    make_db_entry(value, library, version, file_path, function_name)
                 )
             elif feat_type == "function_name":
                 # Future-proof: a minimal extractor may emit function names explicitly.
                 entries.append(
-                    {
-                        "string": value,
-                        "library_name": library,
-                        "library_version": version,
-                        "file_path": file_path,
-                        "function_name": value,
-                        "line_number": None,
-                    }
+                    make_db_entry(value, library, version, file_path, value)
                 )
                 explicit_function_names.add(value)
 
@@ -403,14 +424,7 @@ def parse(
         if self.emit_function_names:
             for fn in function_names - explicit_function_names:
                 entries.append(
-                    {
-                        "string": fn,
-                        "library_name": library,
-                        "library_version": version,
-                        "file_path": None,
-                        "function_name": fn,
-                        "line_number": None,
-                    }
+                    make_db_entry(fn, library, version, None, fn)
                 )
 
         if self.deduplicate:
@@ -422,7 +436,11 @@ def parse(
                     seen[key] = entry
             entries = list(seen.values())
 
-        return entries
+        return ParseResult(
+            entries=entries,
+            num_objects=len(objects),
+            num_functions=len(function_names),
+        )
 
     def write(
         self,
@@ -448,37 +466,6 @@ def write(
             "total_entries": len(entries),
         }
 
-    def convert(
-        self,
-        jh_text: str,
-        library: str,
-        version: str,
-        output_path: pathlib.Path,
-    ) -> dict:
-        """Convenience wrapper: parse jh JSONL and write JSONL.gz. Returns counts."""
-        entries = self.parse(jh_text, library, version)
-        return self.write(entries, output_path)
-
-
-def count_jsonl_rows(jh_text: str) -> dict:
-    """Count objects and unique functions referenced in jh JSONL output."""
-    objects: Set[str] = set()
-    functions: Set[str] = set()
-    for line in jh_text.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        try:
-            row = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        file_path = row.get("path")
-        function_name = row.get("function")
-        if function_name:
-            objects.add(file_path)
-            functions.add(function_name)
-    return {"num_objects": len(objects), "num_functions": len(functions)}
-
 
 def build_library(
     library: str,
@@ -537,11 +524,10 @@ def build_library(
             all_jh_parts.append(jh_text)
 
         combined_jh_text = "\n".join(all_jh_parts)
-        row_counts = count_jsonl_rows(combined_jh_text)
-        metrics.num_objects = row_counts["num_objects"]
-        metrics.num_functions = row_counts["num_functions"]
-
-        entries = converter.parse(combined_jh_text, library, version)
+        result = converter.parse(combined_jh_text, library, version)
+        metrics.num_objects = result.num_objects
+        metrics.num_functions = result.num_functions
+        entries = result.entries
         metrics.num_raw_entries = len(entries)
     except UnsupportedPlatformError as exc:
         logger.warning("%s: skipping unsupported library (%s)", library, exc)
@@ -581,26 +567,18 @@ def load_existing_entries(path: pathlib.Path) -> List[dict]:
         if not isinstance(row, dict) or "string" not in row:
             continue
         entries.append(
-            {
-                "string": row.get("string"),
-                "library_name": row.get("library_name"),
-                "library_version": row.get("library_version"),
-                "file_path": row.get("file_path"),
-                "function_name": row.get("function_name"),
-                "line_number": row.get("line_number"),
-            }
+            make_db_entry(
+                row.get("string"),
+                row.get("library_name"),
+                row.get("library_version"),
+                row.get("file_path"),
+                row.get("function_name"),
+                row.get("line_number"),
+            )
         )
     return entries
 
 
-def write_entries(entries: List[dict], output_path: pathlib.Path) -> None:
-    """Write entries to a gzip-compressed JSONL file."""
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    with gzip.open(output_path, "wt", encoding="utf-8") as f:
-        for entry in entries:
-            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
-
-
 def merge_entries(
     new_entries: List[dict],
     existing_entries: List[dict],
diff --git a/tests/test_build_oss_db.py b/tests/test_build_oss_db.py
new file mode 100644
index 000000000..2a3587212
--- /dev/null
+++ b/tests/test_build_oss_db.py
@@ -0,0 +1,359 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gzip
+import json
+import logging
+import pathlib
+import sys
+
+SCRIPTS_DIR = pathlib.Path(__file__).resolve().parent.parent / "scripts"
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import build_oss_db  # noqa: E402
+
+
+SAMPLE_DB_PATH = pathlib.Path(__file__).resolve().parent.parent / "floss" / "qs" / "db" / "data" / "oss"
+
+
+def _capture_warnings(logger_name: str) -> list:
+    """Attach a recording handler at WARNING level to the named logger.
+
+    Returns a list that the caller can read after invoking the system under
+    test. Each call installs an independent handler with its own list, so
+    tests do not interfere with each other.
+    """
+    records: list = []
+    handler = logging.Handler(level=logging.WARNING)
+
+    def emit(record):
+        records.append(record)
+
+    handler.emit = emit  # type: ignore[assignment]
+    logger = logging.getLogger(logger_name)
+    logger.addHandler(handler)
+    logger.setLevel(logging.WARNING)
+    return records
+
+
+def _row(path, function, feat_type, value):
+    return json.dumps({"path": path, "function": function, "type": feat_type, "value": value})
+
+
+# ---------------------------------------------------------------------------
+# make_db_entry
+# ---------------------------------------------------------------------------
+
+
+def test_make_db_entry_uses_standard_schema():
+    e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn")
+    assert e == {
+        "string": "s",
+        "library_name": "lib",
+        "library_version": "1.0",
+        "file_path": "f.c",
+        "function_name": "fn",
+        "line_number": None,
+    }
+
+
+def test_make_db_entry_line_number_default_is_none():
+    e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn")
+    assert e["line_number"] is None
+
+
+def test_make_db_entry_preserves_explicit_line_number():
+    e = build_oss_db.make_db_entry("s", "lib", "1.0", "f.c", "fn", 42)
+    assert e["line_number"] == 42
+
+
+# ---------------------------------------------------------------------------
+# Converter.parse: structure
+# ---------------------------------------------------------------------------
+
+
+def test_parse_returns_parse_result_with_expected_fields():
+    jh = _row("a.c", "foo", "string", "hello")
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert isinstance(result, build_oss_db.ParseResult)
+    assert hasattr(result, "entries")
+    assert hasattr(result, "num_objects")
+    assert hasattr(result, "num_functions")
+
+
+# ---------------------------------------------------------------------------
+# Converter.parse: single-pass counts
+# ---------------------------------------------------------------------------
+
+
+def test_parse_counts_unique_objects_and_functions():
+    # Three rows: two paths, two functions, with one duplicate row.
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("a.c", "foo", "string", "world"),
+            _row("b.c", "bar", "string", "baz"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert result.num_objects == 2
+    assert result.num_functions == 2
+
+
+def test_parse_counts_include_duplicate_rows():
+    # Same row repeated should not inflate the unique counts.
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("a.c", "foo", "string", "hello"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert result.num_objects == 1
+    assert result.num_functions == 1
+
+
+def test_parse_counts_exclude_rows_without_function():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("b.c", None, "string", "ignored"),
+            _row("c.c", "", "string", "ignored"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert result.num_objects == 1
+    assert result.num_functions == 1
+
+
+def test_parse_counts_none_path_counts_as_an_object():
+    # Rows with `path == None` are still attributed to an "object" (a None path),
+    # matching the previous count_jsonl_rows behavior.
+    jh = _row(None, "foo", "string", "hello")
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert result.num_objects == 1
+    assert result.num_functions == 1
+
+
+# ---------------------------------------------------------------------------
+# Converter.parse: entry list
+# ---------------------------------------------------------------------------
+
+
+def test_parse_emits_string_and_synthetic_function_name_entries():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("b.c", "bar", "string", "baz"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    strings = {e["string"] for e in result.entries}
+    # Two real strings plus the two synthesized function-name entries.
+    assert strings == {"hello", "baz", "foo", "bar"}
+    # All entries carry the library/version passed in.
+    for e in result.entries:
+        assert e["library_name"] == "lib"
+        assert e["library_version"] == "1.0"
+        assert e["line_number"] is None
+
+
+def test_parse_dedup_collapses_identical_strings_by_default():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("a.c", "foo", "string", "hello"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert sum(1 for e in result.entries if e["string"] == "hello") == 1
+
+
+def test_parse_dedup_false_keeps_all_rows():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("a.c", "foo", "string", "hello"),
+        ]
+    )
+    converter = build_oss_db.Converter(deduplicate=False)
+    result = converter.parse(jh, "lib", "1.0")
+    # 2 string rows + 1 synthetic function-name row for "foo"
+    # (synthetic rows are deduped by their function_name, not affected by deduplicate).
+    assert len(result.entries) == 3
+    assert sum(1 for e in result.entries if e["string"] == "hello") == 2
+
+
+def test_parse_dedup_false_keeps_all_function_name_rows():
+    # Two distinct function_name rows on different functions, with deduplicate off,
+    # should all be retained (no dedup of the synthetic entries either).
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            _row("a.c", "bar", "string", "world"),
+        ]
+    )
+    converter = build_oss_db.Converter(deduplicate=False)
+    result = converter.parse(jh, "lib", "1.0")
+    # 2 string rows + 2 synthetic function-name rows (foo, bar)
+    assert len(result.entries) == 4
+    strings = sorted(e["string"] for e in result.entries)
+    assert strings == ["bar", "foo", "hello", "world"]
+
+
+def test_parse_emit_function_names_false_skips_synthetic_entries():
+    jh = _row("a.c", "foo", "string", "hello")
+    converter = build_oss_db.Converter(emit_function_names=False)
+    result = converter.parse(jh, "lib", "1.0")
+    assert len(result.entries) == 1
+    assert result.entries[0]["string"] == "hello"
+    assert result.num_functions == 1  # counts are still tracked
+
+
+def test_parse_explicit_function_name_row_is_not_duplicated():
+    jh = _row("a.c", "foo", "function_name", "fn_x")
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    # The explicit function_name row should appear once, and "fn_x" should
+    # NOT be re-emitted as a synthetic entry.
+    fn_entries = [e for e in result.entries if e["string"] == "fn_x"]
+    assert len(fn_entries) == 1
+    assert fn_entries[0]["function_name"] == "fn_x"
+
+
+# ---------------------------------------------------------------------------
+# Converter.parse: error tolerance
+# ---------------------------------------------------------------------------
+
+
+def test_parse_skips_malformed_jsonl_lines():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            "this is not json",
+            _row("b.c", "bar", "string", "world"),
+        ]
+    )
+    records = _capture_warnings("build_oss_db")
+    converter = build_oss_db.Converter()
+    result = converter.parse(jh, "lib", "1.0")
+    # Counts still cover the well-formed rows.
+    assert result.num_objects == 2
+    assert result.num_functions == 2
+    # A warning was logged for the bad line.
+    assert any("malformed" in record.message.lower() for record in records)
+
+
+def test_parse_skips_empty_lines():
+    jh = "\n".join(
+        [
+            _row("a.c", "foo", "string", "hello"),
+            "",
+            "   ",
+            _row("b.c", "bar", "string", "world"),
+        ]
+    )
+    result = build_oss_db.Converter().parse(jh, "lib", "1.0")
+    assert result.num_objects == 2
+    assert result.num_functions == 2
+
+
+# ---------------------------------------------------------------------------
+# load_existing_entries
+# ---------------------------------------------------------------------------
+
+
+def _write_gz_jsonl(path: pathlib.Path, rows) -> None:
+    """Write each row to a gzipped JSONL file, preserving its raw form.
+
+    Pass dicts to be JSON-serialized, or pass pre-serialized strings to inject
+    malformed/non-JSON content for negative-path tests.
+    """
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        for row in rows:
+            if isinstance(row, str):
+                f.write(row + "\n")
+            else:
+                f.write(json.dumps(row) + "\n")
+
+
+def test_load_existing_entries_round_trips_make_db_entry(tmp_path):
+    path = tmp_path / "lib.jsonl.gz"
+    entries = [
+        build_oss_db.make_db_entry("s1", "lib", "1.0", "f.c", "fn"),
+        build_oss_db.make_db_entry("s2", "lib", "1.0", None, None, 5),
+    ]
+    _write_gz_jsonl(path, entries)
+    assert build_oss_db.load_existing_entries(path) == entries
+
+
+def test_load_existing_entries_missing_file_returns_empty():
+    path = pathlib.Path("/nonexistent/path/to/lib.jsonl.gz")
+    assert build_oss_db.load_existing_entries(path) == []
+
+
+def test_load_existing_entries_skips_malformed_lines(tmp_path):
+    path = tmp_path / "lib.jsonl.gz"
+    _write_gz_jsonl(
+        path,
+        [
+            build_oss_db.make_db_entry("s1", "lib", "1.0", "f.c", "fn"),
+            "this is not json",
+            build_oss_db.make_db_entry("s2", "lib", "1.0", "f.c", "fn"),
+        ],
+    )
+    records = _capture_warnings("build_oss_db")
+    loaded = build_oss_db.load_existing_entries(path)
+    assert len(loaded) == 2
+    assert [e["string"] for e in loaded] == ["s1", "s2"]
+    assert any("malformed" in record.message.lower() for record in records)
+
+
+def test_load_existing_entries_skips_non_dict_and_missing_string(tmp_path):
+    path = tmp_path / "lib.jsonl.gz"
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        f.write(json.dumps([1, 2, 3]) + "\n")  # non-dict
+        f.write(json.dumps({"library_name": "lib"}) + "\n")  # missing "string"
+        f.write(json.dumps({"string": "ok", "library_name": "lib"}) + "\n")
+    loaded = build_oss_db.load_existing_entries(path)
+    assert len(loaded) == 1
+    assert loaded[0]["string"] == "ok"
+
+
+def test_load_existing_entries_ignores_unknown_keys_and_fills_missing(tmp_path):
+    path = tmp_path / "lib.jsonl.gz"
+    _write_gz_jsonl(
+        path,
+        [
+            {"string": "s", "library_name": "l", "extra_key": "ignored"},
+        ],
+    )
+    loaded = build_oss_db.load_existing_entries(path)
+    assert loaded == [
+        {
+            "string": "s",
+            "library_name": "l",
+            "library_version": None,
+            "file_path": None,
+            "function_name": None,
+            "line_number": None,
+        }
+    ]
+
+
+def test_load_existing_entries_handles_empty_file(tmp_path):
+    path = tmp_path / "empty.jsonl.gz"
+    path.write_bytes(b"")
+    # Empty file is not valid gzip; load_existing_entries should warn and return [].
+    assert build_oss_db.load_existing_entries(path) == []

From da23b8cb31b595af9edeb087053ffc6b237ef9a7 Mon Sep 17 00:00:00 2001
From: lakshit verma <vermalucky2004@gmail.com>
Date: Wed, 8 Jul 2026 05:36:41 +0530
Subject: [PATCH 17/19] test: rename test_build_oss_db.py to
 test_qs_build_oss_db.py

Follow the test_qs_* naming so it is selected by 'pytest -q qs'.
---
 tests/{test_build_oss_db.py => test_qs_build_oss_db.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename tests/{test_build_oss_db.py => test_qs_build_oss_db.py} (100%)

diff --git a/tests/test_build_oss_db.py b/tests/test_qs_build_oss_db.py
similarity index 100%
rename from tests/test_build_oss_db.py
rename to tests/test_qs_build_oss_db.py

From fff84063c7aa8741147398252ef06a01890848d3 Mon Sep 17 00:00:00 2001
From: lakshit verma <vermalucky2004@gmail.com>
Date: Wed, 8 Jul 2026 19:46:26 +0530
Subject: [PATCH 18/19] refactor(oss-db): address review feedback; add
 orchestration tests

scripts/build_oss_db.py:
- Drop unused Iterable (typing) and field (dataclasses) imports.
- Drop unused env parameter from run(); none of the four call sites
  pass it.
- Move 'import time' to the top of the module; no lazy-loading
  justification.
- Clean up the awkward config.get('libraries',) call shape.
- Refuse to extract when the vcpkg .list file is missing instead of
  silently scanning the entire triplet lib/ directory (which would
  attribute other packages' strings/functions to the wrong library).
  Log the candidate files we deliberately skipped.
- Fix exit-code logic: with --continue-on-error, return 1 when every
  library failed so the CI step doesn't silently go green on a total
  pipeline failure (broken vcpkg, wrong jh path, etc.).

tests/test_qs_build_oss_db.py:
- Add unit tests for merge_entries (new wins on collision, dedup
  off, both empty, single-side cases).
- Add orchestration tests for main() that stub build_library/Vcpkg/
  JHExtractor: per-library output, merge with pre-existing databases,
  pre-existing libraries not in the current run are preserved,
  --continue-on-error + partial success exits 0,
  --continue-on-error + all failures exits 1, no --continue-on-error +
  any failure exits 1, build_metrics.json is written with the
  expected counts.

floss/qs/db/data/oss/readme.md:
- Add a short 'Why these build parameters' section that justifies
  the x64-windows-static / msvc143 / release choice (PE target;
  toolchain-version impact expected to be marginal) and documents
  the cross-platform string-overlap findings (similar totals, low
  per-string overlap due to ISA byte runs, section names, symbol
  conventions). Stay Windows-only since FLOSS targets PE.

34 tests passing.
---
 floss/qs/db/data/oss/readme.md |   6 +
 scripts/build_oss_db.py        |  57 ++++---
 tests/test_qs_build_oss_db.py  | 304 +++++++++++++++++++++++++++++++++
 3 files changed, 348 insertions(+), 19 deletions(-)

diff --git a/floss/qs/db/data/oss/readme.md b/floss/qs/db/data/oss/readme.md
index 32e58a5e7..1e93c6b58 100644
--- a/floss/qs/db/data/oss/readme.md
+++ b/floss/qs/db/data/oss/readme.md
@@ -51,6 +51,12 @@ Use the triplet `x64-windows-static` to build static archives (.lib files that a
 PS > C:\vcpkg\vcpkg.exe install --triplet x64-windows-static zlib
 ```
 
+### Why these build parameters
+
+We use `triplet=x64-windows-static`, `compiler=msvc143`, `profile=release` to match FLOSS's primary target (PE binaries on Windows). MSVC v143 was the current Visual Studio toolchain at the time the databases were first generated; switching to a newer toolchain would be expected to shift extracted string counts only marginally, since most strings come from source-level literals and symbol names.
+
+A cross-platform check (macOS arm64 vs. MinGW x86_64) showed total counts stay similar but per-string overlap is low: most non-overlap is random printable byte runs in compiled code (which differ per ISA), section/segment names (`.rdata` vs. `__TEXT,__cstring`), and symbol conventions (Mach-O prefixes C symbols with `_`, COFF does not). We keep the databases Windows-only since FLOSS targets PE.
+
 ### Extract features via jh
 
 [jh](https://github.com/williballenthin/lancelot/blob/master/bin/src/bin/jh.rs)
diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index ae8164924..980a544c2 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -38,6 +38,7 @@
 import os
 import re
 import sys
+import time
 import gzip
 import json
 import shutil
@@ -45,8 +46,8 @@
 import pathlib
 import argparse
 import subprocess
-from typing import Set, Dict, List, Tuple, Iterable, Optional
-from dataclasses import field, dataclass
+from typing import Set, Dict, List, Tuple, Optional
+from dataclasses import dataclass
 
 logging.basicConfig(
     level=logging.INFO,
@@ -140,17 +141,12 @@ def run(
     cmd: List[str],
     cwd: Optional[pathlib.Path] = None,
     check: bool = True,
-    env: Optional[dict] = None,
 ) -> subprocess.CompletedProcess:
     """Run a subprocess and return its output."""
     logger.debug("running: %s", " ".join(cmd))
-    merged_env = os.environ.copy()
-    if env:
-        merged_env.update(env)
     result = subprocess.run(
         cmd,
         cwd=str(cwd) if cwd else None,
-        env=merged_env,
         text=True,
         capture_output=True,
     )
@@ -247,14 +243,24 @@ def find_package_libs(self, library: str, triplet: str) -> List[pathlib.Path]:
             list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)]
 
         if not list_files:
-            # Fallback: scan the whole lib directory. This is less precise but works
-            # when the .list file cannot be located.
-            logger.warning(
-                "could not find vcpkg info .list for %s:%s; falling back to lib-dir scan",
+            # The .list file is the only authoritative way to know which static
+            # libraries belong to this package. vcpkg installs every package's
+            # libs into the same shared <triplet>/lib/ directory, so a blind
+            # scan of that directory would attribute other packages'
+            # strings/functions to this library. Refuse to extract instead of
+            # silently polluting the database, and surface the files we would
+            # have wrongly included for diagnostics.
+            candidate_files = self._find_all_static_libs(triplet)
+            logger.error(
+                "could not find vcpkg info .list for %s:%s; "
+                "refusing to extract to avoid misattributing %d other-package "
+                "library file(s): %s",
                 library,
                 triplet,
+                len(candidate_files),
+                ", ".join(p.name for p in candidate_files),
             )
-            return self._find_all_static_libs(triplet)
+            return []
 
         lib_paths: List[pathlib.Path] = []
         for list_file in list_files:
@@ -479,8 +485,6 @@ def build_library(
     The returned entries are not yet written to disk; the caller is responsible
     for cross-library deduplication and final file emission.
     """
-    import time
-
     start = time.time()
     metrics = LibraryMetrics(
         library=library,
@@ -663,9 +667,7 @@ def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
         "triplet": config.get("triplet"),
         "compiler": config.get("compiler"),
         "profile": config.get("profile"),
-        "libraries": config.get(
-            "libraries",
-        ),
+        "libraries": config.get("libraries"),
     }
 
     parser = argparse.ArgumentParser(
@@ -834,9 +836,26 @@ def _lib_name_from_path(p: pathlib.Path) -> str:
     logger.info("wrote metrics to %s", metrics_path)
 
     if failed:
-        logger.error("one or more libraries failed to build")
-        if config.continue_on_error:
+        successful = sum(1 for m in metrics if not m.error)
+        if config.continue_on_error and successful > 0:
+            # Partial success: --continue-on-error let us build some libraries.
+            # The CI will pick up the updated databases and a follow-up run can
+            # retry the failed ones.
+            logger.warning(
+                "%d/%d libraries failed; exiting 0 because --continue-on-error is set",
+                failed,
+                len(metrics),
+            )
             return 0
+        # Either --continue-on-error was not set, or every library failed. In
+        # the latter case we cannot let the workflow step go green: nothing was
+        # produced, so a missing build (broken vcpkg, wrong jh path, etc.)
+        # would be silent.
+        logger.error(
+            "one or more libraries failed to build (failed=%d, total=%d)",
+            failed,
+            len(metrics),
+        )
         return 1
     return 0
 
diff --git a/tests/test_qs_build_oss_db.py b/tests/test_qs_build_oss_db.py
index 2a3587212..53d0cbfc5 100644
--- a/tests/test_qs_build_oss_db.py
+++ b/tests/test_qs_build_oss_db.py
@@ -357,3 +357,307 @@ def test_load_existing_entries_handles_empty_file(tmp_path):
     path.write_bytes(b"")
     # Empty file is not valid gzip; load_existing_entries should warn and return [].
     assert build_oss_db.load_existing_entries(path) == []
+
+
+# ---------------------------------------------------------------------------
+# merge_entries
+# ---------------------------------------------------------------------------
+
+
+def _entry(string, library="lib", version="1.0", function_name="fn"):
+    return build_oss_db.make_db_entry(string, library, version, "f.c", function_name)
+
+
+def test_merge_entries_new_wins_on_collision_when_dedup():
+    new = [_entry("hello", version="2.0"), _entry("world", version="2.0")]
+    existing = [_entry("hello", version="1.0"), _entry("other", version="1.0")]
+    merged = build_oss_db.merge_entries(new, existing, deduplicate=True)
+
+    by_string = {e["string"]: e for e in merged}
+    # New "hello" wins (version 2.0), "world" is new, "other" is from existing.
+    assert by_string["hello"]["library_version"] == "2.0"
+    assert by_string["world"]["library_version"] == "2.0"
+    assert by_string["other"]["library_version"] == "1.0"
+    assert len(merged) == 3
+
+
+def test_merge_entries_dedup_false_keeps_duplicates():
+    new = [_entry("hello"), _entry("world")]
+    existing = [_entry("hello"), _entry("other")]
+    merged = build_oss_db.merge_entries(new, existing, deduplicate=False)
+    # All four rows preserved. Without dedup the function concatenates
+    # existing first, then new; the order is purely cosmetic (the loader
+    # indexes by string).
+    assert [e["string"] for e in merged] == ["hello", "other", "hello", "world"]
+
+
+def test_merge_entries_both_empty_returns_empty():
+    assert build_oss_db.merge_entries([], [], deduplicate=True) == []
+    assert build_oss_db.merge_entries([], [], deduplicate=False) == []
+
+
+def test_merge_entries_only_new_returns_copy_of_new():
+    new = [_entry("a"), _entry("b")]
+    result = build_oss_db.merge_entries(new, [], deduplicate=True)
+    assert result == new
+    assert result is not new  # callers rely on a fresh list
+
+
+def test_merge_entries_only_existing_returns_copy_of_existing():
+    existing = [_entry("a"), _entry("b")]
+    result = build_oss_db.merge_entries([], existing, deduplicate=True)
+    assert result == existing
+    assert result is not existing
+
+
+# ---------------------------------------------------------------------------
+# main() orchestration
+# ---------------------------------------------------------------------------
+
+
+class _FakeVcpkg:
+    """Stand-in for Vcpkg: records the install calls but does nothing."""
+
+    def __init__(self, *args, **kwargs):
+        self.installed = []
+
+    def install(self, library, triplet):
+        self.installed.append((library, triplet))
+
+    def get_installed_version(self, library, triplet):
+        return "1.0#1"
+
+    def find_package_libs(self, library, triplet):
+        return []
+
+
+class _FakeJH:
+    def __init__(self, *args, **kwargs):
+        pass
+
+
+def _stub_build_library(library, config, vcpkg, jh, converter):
+    """Replacement for build_library that returns the library's name as its only entry."""
+    metrics = build_oss_db.LibraryMetrics(
+        library=library,
+        version="1.0#1",
+        triplet=config.triplet,
+        num_objects=0,
+        num_functions=0,
+        num_raw_entries=1,
+        total_entries=1,
+        duration_seconds=0.0,
+    )
+    entry = build_oss_db.make_db_entry(
+        f"hello-from-{library}",
+        library,
+        "1.0#1",
+        "f.c",
+        f"fn_{library}",
+    )
+    return metrics, [entry]
+
+
+def _invoke_main(monkeypatch, output_dir, libraries, *, continue_on_error=False, existing=None):
+    """Invoke main() with build_library stubbed to return one entry per library.
+
+    ``existing`` is a dict of {library_name: [entry, ...]} to write to the
+    output dir as if they were previously built.
+    """
+    if existing:
+        for lib, entries in existing.items():
+            path = output_dir / f"{lib}.jsonl.gz"
+            with gzip.open(path, "wt", encoding="utf-8") as f:
+                for e in entries:
+                    f.write(json.dumps(e) + "\n")
+
+    # Replace heavy machinery with no-op fakes; stub build_library so it
+    # doesn't actually call vcpkg or jh.
+    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
+    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
+    monkeypatch.setattr(build_oss_db, "build_library", _stub_build_library)
+
+    argv = [
+        "--triplet",
+        "x64-windows-static",
+        "--compiler",
+        "msvc143",
+        "--profile",
+        "release",
+        "--output-dir",
+        str(output_dir),
+        "--libraries",
+        *libraries,
+    ]
+    if continue_on_error:
+        argv.append("--continue-on-error")
+
+    return build_oss_db.main(argv)
+
+
+def _read_gz_jsonl(path: pathlib.Path):
+    with gzip.open(path, "rt", encoding="utf-8") as f:
+        return [json.loads(line) for line in f if line.strip()]
+
+
+def test_main_writes_one_database_per_library(tmp_path, monkeypatch):
+    rc = _invoke_main(monkeypatch, tmp_path, ["zlib", "curl"])
+    assert rc == 0
+    for lib in ("zlib", "curl"):
+        entries = _read_gz_jsonl(tmp_path / f"{lib}.jsonl.gz")
+        assert len(entries) == 1
+        assert entries[0]["string"] == f"hello-from-{lib}"
+        assert entries[0]["library_name"] == lib
+
+
+def test_main_merges_existing_database_with_fresh_entries(tmp_path, monkeypatch):
+    existing_entry = build_oss_db.make_db_entry(
+        "old-string", "zlib", "0.9#1", "f.c", "old_fn"
+    )
+    rc = _invoke_main(
+        monkeypatch,
+        tmp_path,
+        ["zlib"],
+        existing={"zlib": [existing_entry]},
+    )
+    assert rc == 0
+    entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz")
+    strings = {e["string"] for e in entries}
+    # Old entry from disk is preserved; new entry from the stubbed build is added.
+    assert strings == {"old-string", "hello-from-zlib"}
+
+
+def test_main_preserves_existing_libraries_not_in_current_run(tmp_path, monkeypatch):
+    # Pre-existing database for a library we are NOT rebuilding this run.
+    preexisting = [build_oss_db.make_db_entry("preexisting", "other", "1.0", "f.c", "fn")]
+    rc = _invoke_main(
+        monkeypatch,
+        tmp_path,
+        ["zlib"],
+        existing={"other": preexisting},
+    )
+    assert rc == 0
+    # "other" database was not rewritten; its content is unchanged.
+    entries = _read_gz_jsonl(tmp_path / "other.jsonl.gz")
+    assert entries == preexisting
+    # "zlib" was rebuilt.
+    zlib_entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz")
+    assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"}
+
+
+def test_main_exits_zero_on_partial_success_with_continue_on_error(tmp_path, monkeypatch):
+    def stub_partial(library, config, vcpkg, jh, converter):
+        if library == "broken":
+            metrics = build_oss_db.LibraryMetrics(
+                library=library,
+                version="unknown",
+                triplet=config.triplet,
+                error="boom",
+            )
+            return metrics, []
+        return _stub_build_library(library, config, vcpkg, jh, converter)
+
+    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
+    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
+    monkeypatch.setattr(build_oss_db, "build_library", stub_partial)
+
+    argv = [
+        "--triplet",
+        "x64-windows-static",
+        "--compiler",
+        "msvc143",
+        "--profile",
+        "release",
+        "--output-dir",
+        str(tmp_path),
+        "--libraries",
+        "zlib",
+        "broken",
+        "--continue-on-error",
+    ]
+    rc = build_oss_db.main(argv)
+    # At least one library succeeded, so the workflow should see a green step.
+    assert rc == 0
+    # The successful library's database was still written.
+    zlib_entries = _read_gz_jsonl(tmp_path / "zlib.jsonl.gz")
+    assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"}
+
+
+def test_main_exits_nonzero_when_all_libraries_fail_with_continue_on_error(tmp_path, monkeypatch):
+    def stub_all_fail(library, config, vcpkg, jh, converter):
+        metrics = build_oss_db.LibraryMetrics(
+            library=library,
+            version="unknown",
+            triplet=config.triplet,
+            error="boom",
+        )
+        return metrics, []
+
+    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
+    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
+    monkeypatch.setattr(build_oss_db, "build_library", stub_all_fail)
+
+    argv = [
+        "--triplet",
+        "x64-windows-static",
+        "--compiler",
+        "msvc143",
+        "--profile",
+        "release",
+        "--output-dir",
+        str(tmp_path),
+        "--libraries",
+        "broken1",
+        "broken2",
+        "--continue-on-error",
+    ]
+    rc = build_oss_db.main(argv)
+    # Everything failed, even with --continue-on-error: must exit non-zero so
+    # the CI step doesn't silently go green on a total pipeline failure.
+    assert rc == 1
+
+
+def test_main_exits_nonzero_on_any_failure_without_continue_on_error(tmp_path, monkeypatch):
+    def stub_one_fail(library, config, vcpkg, jh, converter):
+        if library == "broken":
+            metrics = build_oss_db.LibraryMetrics(
+                library=library,
+                version="unknown",
+                triplet=config.triplet,
+                error="boom",
+            )
+            return metrics, []
+        return _stub_build_library(library, config, vcpkg, jh, converter)
+
+    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
+    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
+    monkeypatch.setattr(build_oss_db, "build_library", stub_one_fail)
+
+    argv = [
+        "--triplet",
+        "x64-windows-static",
+        "--compiler",
+        "msvc143",
+        "--profile",
+        "release",
+        "--output-dir",
+        str(tmp_path),
+        "--libraries",
+        "zlib",
+        "broken",
+    ]
+    rc = build_oss_db.main(argv)
+    assert rc == 1
+
+
+def test_main_writes_build_metrics_summary(tmp_path, monkeypatch):
+    rc = _invoke_main(monkeypatch, tmp_path, ["zlib", "curl"])
+    assert rc == 0
+    summary = json.loads((tmp_path / "build_metrics.json").read_text())
+    assert summary["triplet"] == "x64-windows-static"
+    assert summary["compiler"] == "msvc143"
+    assert summary["profile"] == "release"
+    assert summary["successful"] == 2
+    assert summary["failed"] == 0
+    names = {m["library"] for m in summary["libraries"]}
+    assert names == {"zlib", "curl"}

From 3821279d1aa186dbd79e46b5e7680b3b8984fc95 Mon Sep 17 00:00:00 2001
From: lakshit verma <vermalucky2004@gmail.com>
Date: Wed, 8 Jul 2026 20:13:25 +0530
Subject: [PATCH 19/19] refactor(oss-db): remove monkeypatching from
 orchestration tests

---
 scripts/build_oss_db.py       | 196 ++++++++++++++++++----------------
 tests/test_qs_build_oss_db.py | 148 ++++++++++---------------
 2 files changed, 160 insertions(+), 184 deletions(-)

diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py
index 980a544c2..2d562b7a0 100644
--- a/scripts/build_oss_db.py
+++ b/scripts/build_oss_db.py
@@ -46,7 +46,7 @@
 import pathlib
 import argparse
 import subprocess
-from typing import Set, Dict, List, Tuple, Optional
+from typing import Callable, Set, Dict, List, Tuple, Optional
 from dataclasses import dataclass
 
 logging.basicConfig(
@@ -641,6 +641,109 @@ def write_library_database(
     return metrics
 
 
+def run_build(
+    config: BuildConfig,
+    vcpkg: Vcpkg,
+    jh: JHExtractor,
+    converter: Converter,
+    build_library_fn: Callable[[str, BuildConfig, Vcpkg, JHExtractor, Converter], Tuple[LibraryMetrics, List[dict]]] = build_library,
+) -> int:
+    """Build and write databases for the configured libraries.
+
+    This is split out from `main()` so tests can inject fakes directly without
+    monkeypatching module globals.
+    """
+    config.output_dir.mkdir(parents=True, exist_ok=True)
+
+    metrics: List[LibraryMetrics] = []
+    per_library_new: Dict[str, List[dict]] = {}
+    failed = False
+    for library in config.libraries:
+        metric, entries = build_library_fn(library, config, vcpkg, jh, converter)
+        metrics.append(metric)
+        # Even on error, preserve any partial entries so they aren't lost.
+        per_library_new[library] = entries
+        if metric.error:
+            failed = True
+            if not config.continue_on_error:
+                break
+
+    # Discover all existing .jsonl.gz databases in the output directory. These
+    # include libraries we are rebuilding (whose fresh entries will be merged
+    # in) and libraries we are leaving alone. Strings are NOT deduped across
+    # libraries: a string that appears in both zlib and curl (e.g. when zlib
+    # is vendored into curl) stays in both databases. The query tagger already
+    # emits one #library tag per matching database, so the consumer can see
+    # the overlap directly.
+    existing_files = sorted(config.output_dir.glob("*.jsonl.gz"))
+
+    def _lib_name_from_path(p: pathlib.Path) -> str:
+        # p.name looks like "<lib>.jsonl.gz"; Path.stem would only strip ".gz".
+        suffix = ".jsonl.gz"
+        if p.name.endswith(suffix):
+            return p.name[: -len(suffix)]
+        return p.stem
+
+    merged: Dict[str, List[dict]] = {}
+    for path in existing_files:
+        lib = _lib_name_from_path(path)
+        existing = load_existing_entries(path)
+        if lib in per_library_new:
+            merged[lib] = merge_entries(per_library_new[lib], existing, config.deduplicate)
+        elif existing:
+            merged[lib] = existing
+
+    # Libraries being built for the first time (no pre-existing file).
+    for lib, new_entries in per_library_new.items():
+        if lib not in merged:
+            merged[lib] = list(new_entries)
+
+    # Write rebuilt libraries and update their metrics.
+    for metric in metrics:
+        if metric.error:
+            logger.warning("%s: skipping database write due to build error", metric.library)
+            continue
+        entries = merged.get(metric.library, [])
+        write_library_database(metric, entries, config.output_dir, converter)
+
+    summary = {
+        "triplet": config.triplet,
+        "compiler": config.compiler,
+        "profile": config.profile,
+        "libraries": [m.as_dict() for m in metrics],
+        "successful": sum(1 for m in metrics if not m.error),
+        "failed": sum(1 for m in metrics if m.error),
+    }
+
+    metrics_path = config.output_dir / "build_metrics.json"
+    metrics_path.write_text(json.dumps(summary, indent=2))
+    logger.info("wrote metrics to %s", metrics_path)
+
+    if failed:
+        successful = sum(1 for m in metrics if not m.error)
+        if config.continue_on_error and successful > 0:
+            # Partial success: --continue-on-error let us build some libraries.
+            # The CI will pick up the updated databases and a follow-up run can
+            # retry the failed ones.
+            logger.warning(
+                "%d/%d libraries failed; exiting 0 because --continue-on-error is set",
+                failed,
+                len(metrics),
+            )
+            return 0
+        # Either --continue-on-error was not set, or every library failed. In
+        # the latter case we cannot let the workflow step go green: nothing was
+        # produced, so a missing build (broken vcpkg, wrong jh path, etc.)
+        # would be silent.
+        logger.error(
+            "one or more libraries failed to build (failed=%d, total=%d)",
+            failed,
+            len(metrics),
+        )
+        return 1
+    return 0
+
+
 def load_config(path: pathlib.Path) -> dict:
     """Load build configuration from a JSON file."""
     data = json.loads(path.read_text())
@@ -768,96 +871,7 @@ def main(argv: Optional[List[str]] = None) -> int:
         deduplicate=config.deduplicate,
     )
 
-    config.output_dir.mkdir(parents=True, exist_ok=True)
-
-    metrics: List[LibraryMetrics] = []
-    per_library_new: Dict[str, List[dict]] = {}
-    failed = False
-    for library in config.libraries:
-        metric, entries = build_library(library, config, vcpkg, jh, converter)
-        metrics.append(metric)
-        # Even on error, preserve any partial entries so they aren't lost.
-        per_library_new[library] = entries
-        if metric.error:
-            failed = True
-            if not config.continue_on_error:
-                break
-
-    # Discover all existing .jsonl.gz databases in the output directory. These
-    # include libraries we are rebuilding (whose fresh entries will be merged
-    # in) and libraries we are leaving alone. Strings are NOT deduped across
-    # libraries: a string that appears in both zlib and curl (e.g. when zlib
-    # is vendored into curl) stays in both databases. The query tagger already
-    # emits one #library tag per matching database, so the consumer can see
-    # the overlap directly.
-    existing_files = sorted(config.output_dir.glob("*.jsonl.gz"))
-    metric_by_lib: Dict[str, LibraryMetrics] = {m.library: m for m in metrics}
-
-    def _lib_name_from_path(p: pathlib.Path) -> str:
-        # p.name looks like "<lib>.jsonl.gz"; Path.stem would only strip ".gz".
-        suffix = ".jsonl.gz"
-        if p.name.endswith(suffix):
-            return p.name[: -len(suffix)]
-        return p.stem
-
-    merged: Dict[str, List[dict]] = {}
-    for path in existing_files:
-        lib = _lib_name_from_path(path)
-        existing = load_existing_entries(path)
-        if lib in per_library_new:
-            merged[lib] = merge_entries(per_library_new[lib], existing, config.deduplicate)
-        elif existing:
-            merged[lib] = existing
-
-    # Libraries being built for the first time (no pre-existing file).
-    for lib, new_entries in per_library_new.items():
-        if lib not in merged:
-            merged[lib] = list(new_entries)
-
-    # Write rebuilt libraries and update their metrics.
-    for metric in metrics:
-        if metric.error:
-            logger.warning("%s: skipping database write due to build error", metric.library)
-            continue
-        entries = merged.get(metric.library, [])
-        write_library_database(metric, entries, config.output_dir, converter)
-
-    summary = {
-        "triplet": config.triplet,
-        "compiler": config.compiler,
-        "profile": config.profile,
-        "libraries": [m.as_dict() for m in metrics],
-        "successful": sum(1 for m in metrics if not m.error),
-        "failed": sum(1 for m in metrics if m.error),
-    }
-
-    metrics_path = config.output_dir / "build_metrics.json"
-    metrics_path.write_text(json.dumps(summary, indent=2))
-    logger.info("wrote metrics to %s", metrics_path)
-
-    if failed:
-        successful = sum(1 for m in metrics if not m.error)
-        if config.continue_on_error and successful > 0:
-            # Partial success: --continue-on-error let us build some libraries.
-            # The CI will pick up the updated databases and a follow-up run can
-            # retry the failed ones.
-            logger.warning(
-                "%d/%d libraries failed; exiting 0 because --continue-on-error is set",
-                failed,
-                len(metrics),
-            )
-            return 0
-        # Either --continue-on-error was not set, or every library failed. In
-        # the latter case we cannot let the workflow step go green: nothing was
-        # produced, so a missing build (broken vcpkg, wrong jh path, etc.)
-        # would be silent.
-        logger.error(
-            "one or more libraries failed to build (failed=%d, total=%d)",
-            failed,
-            len(metrics),
-        )
-        return 1
-    return 0
+    return run_build(config, vcpkg, jh, converter)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_qs_build_oss_db.py b/tests/test_qs_build_oss_db.py
index 53d0cbfc5..9ee258af3 100644
--- a/tests/test_qs_build_oss_db.py
+++ b/tests/test_qs_build_oss_db.py
@@ -458,12 +458,30 @@ def _stub_build_library(library, config, vcpkg, jh, converter):
     return metrics, [entry]
 
 
-def _invoke_main(monkeypatch, output_dir, libraries, *, continue_on_error=False, existing=None):
-    """Invoke main() with build_library stubbed to return one entry per library.
+def _make_config(output_dir, libraries, *, continue_on_error=False):
+    return build_oss_db.BuildConfig(
+        triplet="x64-windows-static",
+        compiler="msvc143",
+        profile="release",
+        libraries=list(libraries),
+        output_dir=output_dir,
+        vcpkg_root=None,
+        jh_path=None,
+        lancelot_dir=None,
+        emit_function_names=True,
+        deduplicate=True,
+        continue_on_error=continue_on_error,
+    )
+
+
+def _invoke_run_build(output_dir, libraries, *, continue_on_error=False, existing=None, build_library_fn=_stub_build_library):
+    """Invoke run_build() with build_library stubbed to return one entry per library.
 
     ``existing`` is a dict of {library_name: [entry, ...]} to write to the
     output dir as if they were previously built.
     """
+    config = _make_config(output_dir, libraries, continue_on_error=continue_on_error)
+
     if existing:
         for lib, entries in existing.items():
             path = output_dir / f"{lib}.jsonl.gz"
@@ -471,28 +489,13 @@ def _invoke_main(monkeypatch, output_dir, libraries, *, continue_on_error=False,
                 for e in entries:
                     f.write(json.dumps(e) + "\n")
 
-    # Replace heavy machinery with no-op fakes; stub build_library so it
-    # doesn't actually call vcpkg or jh.
-    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
-    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
-    monkeypatch.setattr(build_oss_db, "build_library", _stub_build_library)
-
-    argv = [
-        "--triplet",
-        "x64-windows-static",
-        "--compiler",
-        "msvc143",
-        "--profile",
-        "release",
-        "--output-dir",
-        str(output_dir),
-        "--libraries",
-        *libraries,
-    ]
-    if continue_on_error:
-        argv.append("--continue-on-error")
-
-    return build_oss_db.main(argv)
+    return build_oss_db.run_build(
+        config,
+        _FakeVcpkg(),
+        _FakeJH(),
+        build_oss_db.Converter(),
+        build_library_fn=build_library_fn,
+    )
 
 
 def _read_gz_jsonl(path: pathlib.Path):
@@ -500,8 +503,8 @@ def _read_gz_jsonl(path: pathlib.Path):
         return [json.loads(line) for line in f if line.strip()]
 
 
-def test_main_writes_one_database_per_library(tmp_path, monkeypatch):
-    rc = _invoke_main(monkeypatch, tmp_path, ["zlib", "curl"])
+def test_main_writes_one_database_per_library(tmp_path):
+    rc = _invoke_run_build(tmp_path, ["zlib", "curl"])
     assert rc == 0
     for lib in ("zlib", "curl"):
         entries = _read_gz_jsonl(tmp_path / f"{lib}.jsonl.gz")
@@ -510,12 +513,11 @@ def test_main_writes_one_database_per_library(tmp_path, monkeypatch):
         assert entries[0]["library_name"] == lib
 
 
-def test_main_merges_existing_database_with_fresh_entries(tmp_path, monkeypatch):
+def test_main_merges_existing_database_with_fresh_entries(tmp_path):
     existing_entry = build_oss_db.make_db_entry(
         "old-string", "zlib", "0.9#1", "f.c", "old_fn"
     )
-    rc = _invoke_main(
-        monkeypatch,
+    rc = _invoke_run_build(
         tmp_path,
         ["zlib"],
         existing={"zlib": [existing_entry]},
@@ -527,11 +529,10 @@ def test_main_merges_existing_database_with_fresh_entries(tmp_path, monkeypatch)
     assert strings == {"old-string", "hello-from-zlib"}
 
 
-def test_main_preserves_existing_libraries_not_in_current_run(tmp_path, monkeypatch):
+def test_main_preserves_existing_libraries_not_in_current_run(tmp_path):
     # Pre-existing database for a library we are NOT rebuilding this run.
     preexisting = [build_oss_db.make_db_entry("preexisting", "other", "1.0", "f.c", "fn")]
-    rc = _invoke_main(
-        monkeypatch,
+    rc = _invoke_run_build(
         tmp_path,
         ["zlib"],
         existing={"other": preexisting},
@@ -545,7 +546,7 @@ def test_main_preserves_existing_libraries_not_in_current_run(tmp_path, monkeypa
     assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"}
 
 
-def test_main_exits_zero_on_partial_success_with_continue_on_error(tmp_path, monkeypatch):
+def test_main_exits_zero_on_partial_success_with_continue_on_error(tmp_path):
     def stub_partial(library, config, vcpkg, jh, converter):
         if library == "broken":
             metrics = build_oss_db.LibraryMetrics(
@@ -557,25 +558,12 @@ def stub_partial(library, config, vcpkg, jh, converter):
             return metrics, []
         return _stub_build_library(library, config, vcpkg, jh, converter)
 
-    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
-    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
-    monkeypatch.setattr(build_oss_db, "build_library", stub_partial)
-
-    argv = [
-        "--triplet",
-        "x64-windows-static",
-        "--compiler",
-        "msvc143",
-        "--profile",
-        "release",
-        "--output-dir",
-        str(tmp_path),
-        "--libraries",
-        "zlib",
-        "broken",
-        "--continue-on-error",
-    ]
-    rc = build_oss_db.main(argv)
+    rc = _invoke_run_build(
+        tmp_path,
+        ["zlib", "broken"],
+        continue_on_error=True,
+        build_library_fn=stub_partial,
+    )
     # At least one library succeeded, so the workflow should see a green step.
     assert rc == 0
     # The successful library's database was still written.
@@ -583,7 +571,7 @@ def stub_partial(library, config, vcpkg, jh, converter):
     assert {e["string"] for e in zlib_entries} == {"hello-from-zlib"}
 
 
-def test_main_exits_nonzero_when_all_libraries_fail_with_continue_on_error(tmp_path, monkeypatch):
+def test_main_exits_nonzero_when_all_libraries_fail_with_continue_on_error(tmp_path):
     def stub_all_fail(library, config, vcpkg, jh, converter):
         metrics = build_oss_db.LibraryMetrics(
             library=library,
@@ -593,31 +581,18 @@ def stub_all_fail(library, config, vcpkg, jh, converter):
         )
         return metrics, []
 
-    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
-    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
-    monkeypatch.setattr(build_oss_db, "build_library", stub_all_fail)
-
-    argv = [
-        "--triplet",
-        "x64-windows-static",
-        "--compiler",
-        "msvc143",
-        "--profile",
-        "release",
-        "--output-dir",
-        str(tmp_path),
-        "--libraries",
-        "broken1",
-        "broken2",
-        "--continue-on-error",
-    ]
-    rc = build_oss_db.main(argv)
+    rc = _invoke_run_build(
+        tmp_path,
+        ["broken1", "broken2"],
+        continue_on_error=True,
+        build_library_fn=stub_all_fail,
+    )
     # Everything failed, even with --continue-on-error: must exit non-zero so
     # the CI step doesn't silently go green on a total pipeline failure.
     assert rc == 1
 
 
-def test_main_exits_nonzero_on_any_failure_without_continue_on_error(tmp_path, monkeypatch):
+def test_main_exits_nonzero_on_any_failure_without_continue_on_error(tmp_path):
     def stub_one_fail(library, config, vcpkg, jh, converter):
         if library == "broken":
             metrics = build_oss_db.LibraryMetrics(
@@ -629,29 +604,16 @@ def stub_one_fail(library, config, vcpkg, jh, converter):
             return metrics, []
         return _stub_build_library(library, config, vcpkg, jh, converter)
 
-    monkeypatch.setattr(build_oss_db, "Vcpkg", _FakeVcpkg)
-    monkeypatch.setattr(build_oss_db, "JHExtractor", _FakeJH)
-    monkeypatch.setattr(build_oss_db, "build_library", stub_one_fail)
-
-    argv = [
-        "--triplet",
-        "x64-windows-static",
-        "--compiler",
-        "msvc143",
-        "--profile",
-        "release",
-        "--output-dir",
-        str(tmp_path),
-        "--libraries",
-        "zlib",
-        "broken",
-    ]
-    rc = build_oss_db.main(argv)
+    rc = _invoke_run_build(
+        tmp_path,
+        ["zlib", "broken"],
+        build_library_fn=stub_one_fail,
+    )
     assert rc == 1
 
 
-def test_main_writes_build_metrics_summary(tmp_path, monkeypatch):
-    rc = _invoke_main(monkeypatch, tmp_path, ["zlib", "curl"])
+def test_main_writes_build_metrics_summary(tmp_path):
+    rc = _invoke_run_build(tmp_path, ["zlib", "curl"])
     assert rc == 0
     summary = json.loads((tmp_path / "build_metrics.json").read_text())
     assert summary["triplet"] == "x64-windows-static"