diff --git a/.github/workflows/build-oss-db.yml b/.github/workflows/build-oss-db.yml new file mode 100644 index 000000000..635ef79e1 --- /dev/null +++ b/.github/workflows/build-oss-db.yml @@ -0,0 +1,88 @@ +name: Build OSS String Databases + +on: + # Rebuild the databases bi-weekly (1st and 15th of each month at 00:00 UTC). + schedule: + - cron: '0 0 1,15 * *' + + # Allow manual runs from the Actions tab. + workflow_dispatch: + +# Cancel any in-progress run when a newer one starts. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + pull-requests: write + +jobs: + build-databases: + name: Build OSS string databases + runs-on: windows-latest + + env: + # Workaround for zydis 3.1.3's CMakeLists.txt requiring an older + # cmake_minimum_required syntax. Remove once Lancelot updates zydis. + CMAKE_POLICY_VERSION_MINIMUM: '3.5' + + steps: + - name: Checkout flare-floss + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + path: flare-floss + lfs: true + + - name: Checkout lancelot + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: williballenthin/lancelot + path: lancelot + lfs: true + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Set up Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Build jh + run: | + cargo build --release -p lancelot-bin + working-directory: lancelot + + - name: Build OSS databases + run: | + python scripts/build_oss_db.py ` + --config floss\qs\db\data\oss\libraries.json ` + --jh-path ..\lancelot\target\release\jh.exe ` + --output-dir floss\qs\db\data\oss ` + --continue-on-error + working-directory: flare-floss + + - name: Show metrics summary + if: success() || failure() + run: | + Get-Content floss\qs\db\data\oss\build_metrics.json + working-directory: flare-floss + + - name: Create Pull Request if databases changed + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 + with: + path: flare-floss + base: quantumstrand + commit-message: 'chore(oss-db): update string databases' + title: 'Update OSS string databases' + body: | + Automated bi-weekly rebuild of the OSS string databases. + + - Triplet: `x64-windows-static` + - Compiler: `msvc143` + - Library list: see `floss/qs/db/data/oss/libraries.json` + + Per-library entry counts and timing are in `floss/qs/db/data/oss/build_metrics.json`. + branch: update-oss-string-databases + delete-branch: true diff --git a/floss/qs/db/data/oss/libraries.json b/floss/qs/db/data/oss/libraries.json new file mode 100644 index 000000000..32bf0c1f7 --- /dev/null +++ b/floss/qs/db/data/oss/libraries.json @@ -0,0 +1,69 @@ +{ + "triplet": "x64-windows-static", + "compiler": "msvc143", + "profile": "release", + "libraries": [ + "openssl", + "capnproto", + "raylib", + "libgit2", + "libxml2", + "libimobiledevice", + "libarchive", + "libmysql", + "libhv", + "cello", + "czmq", + "boost-test", + "wolfssl", + "boost-wave", + "curl", + "poco", + "nng", + "libsrt", + "sail", + "mbedtls", + "boost-log", + "libsndfile", + "libevent", + "libcoap", + "sqlite3", + "sdl2", + "libbson", + "mongoose", + "duktape", + "libpcap", + "expat", + "libuv", + "llhttp", + "jansson", + "tinyfiledialogs", + "hiredis", + "zyre", + "boost-json", + "boost-serialization", + "tre", + "zlib", + "boost-graph", + "apr", + "libev", + "flatcc", + "boost-filesystem", + "boost-iostreams", + "oniguruma", + "libyaml", + "cunit", + "avro-c", + "lz4", + "parson", + "json-c", + "libzip", + "boost-container", + "lzo", + "liblzma", + "boost-chrono", + "lmdb", + "binn", + "boost-thread" + ] +} diff --git a/floss/qs/db/data/oss/readme.md b/floss/qs/db/data/oss/readme.md index 29f21e80d..1c8acfa79 100644 --- a/floss/qs/db/data/oss/readme.md +++ b/floss/qs/db/data/oss/readme.md @@ -94,3 +94,19 @@ These files are then gzip'd: ```console $ gzip -c zlib.jsonl > zlib.jsonl.gz ``` + +The `build_oss_db.py` script automates the steps above and additionally +removes strings that appear in two or more libraries from every database. +A shared string is not a unique identifier for any specific library, so +attributing it would produce false-positive library matches at query time. + +The script also merges into any `.jsonl.gz` already present in the output +directory rather than replacing them. For each library being rebuilt, the +newly-extracted entries are combined with the existing entries (new entries +win on string collisions, which is the right behavior when the library +version changes). Libraries that are not in the current `--libraries` list +are also re-checked against the cross-library dedup and rewritten if the +shared-string set changes. This makes it safe to incrementally add a new +library (e.g. `python build_oss_db.py --libraries newlib`) without losing +the other databases, and safe to bump one library's version while leaving +its previously-committed entries around. diff --git a/floss/qs/db/oss.py b/floss/qs/db/oss.py index 1d0d4c72d..58b5e1dd0 100644 --- a/floss/qs/db/oss.py +++ b/floss/qs/db/oss.py @@ -15,6 +15,11 @@ class OpenSourceString(msgspec.Struct): file_path: str | None = None function_name: str | None = None line_number: int | None = None + # Number of distinct libraries the string was observed in across the full + # OSS database set. count=1 means the string is unique to this library; + # higher counts indicate strings shared across multiple libraries (e.g. + # vendored dependencies or common runtime strings). + count: int = 1 @dataclass diff --git a/scripts/build_oss_db.py b/scripts/build_oss_db.py new file mode 100644 index 000000000..f953b5d3d --- /dev/null +++ b/scripts/build_oss_db.py @@ -0,0 +1,957 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Build OSS string databases from vcpkg static libraries. + +This script automates the "vcpkg & jh" technique described in readme.md: + + 1. install static libraries via vcpkg + 2. extract string features (and function names) via jh + 3. convert to JSONL and compress with gzip + +It is intentionally modular so the underlying extractor (jh today) can be +swapped for a more minimal tool later without rewriting the orchestration. + +After every library is parsed, each string's ``count`` is set to the number +of distinct libraries the string appears in across the full OSS database +set. The count is exposed via OpenSourceString.count so that consumers can +distinguish strings unique to one library (count=1) from strings shared +across many libraries (count>=2). Strings are NOT removed across +libraries: a string observed in both zlib and curl (e.g. when zlib is +vendored into curl) stays in both databases with count=2, rather than +being dropped from both as a "shared" string. +""" + +from __future__ import annotations + +import os +import re +import sys +import gzip +import json +import shutil +import logging +import pathlib +import argparse +import subprocess +from typing import Set, Dict, List, Tuple, Iterable, Optional +from dataclasses import field, dataclass + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", +) +logger = logging.getLogger("build_oss_db") + + +class BuildError(Exception): + """Raised when a single library cannot be built; the caller decides whether to abort or continue.""" + + +class UnsupportedPlatformError(BuildError): + """Raised when a library does not support the target triplet/platform.""" + + +@dataclass(frozen=True) +class BuildConfig: + triplet: str + compiler: str + profile: str + libraries: List[str] + output_dir: pathlib.Path + vcpkg_root: Optional[pathlib.Path] + jh_path: Optional[pathlib.Path] + lancelot_dir: Optional[pathlib.Path] + emit_function_names: bool = True + deduplicate: bool = True + continue_on_error: bool = False + + +@dataclass +class LibraryMetrics: + library: str + version: str + triplet: str + num_objects: int = 0 + num_functions: int = 0 + num_string_entries: int = 0 + num_function_name_entries: int = 0 + num_raw_entries: int = 0 + total_entries: int = 0 + duration_seconds: float = 0.0 + error: Optional[str] = None + + def as_dict(self) -> dict: + return { + "library": self.library, + "version": self.version, + "triplet": self.triplet, + "num_objects": self.num_objects, + "num_functions": self.num_functions, + "num_string_entries": self.num_string_entries, + "num_function_name_entries": self.num_function_name_entries, + "num_raw_entries": self.num_raw_entries, + "total_entries": self.total_entries, + "duration_seconds": round(self.duration_seconds, 2), + "error": self.error, + } + + +def run( + cmd: List[str], + cwd: Optional[pathlib.Path] = None, + check: bool = True, + env: Optional[dict] = None, +) -> subprocess.CompletedProcess: + """Run a subprocess and return its output.""" + logger.debug("running: %s", " ".join(cmd)) + merged_env = os.environ.copy() + if env: + merged_env.update(env) + result = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + env=merged_env, + text=True, + capture_output=True, + ) + if check and result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, + cmd, + output=result.stdout, + stderr=result.stderr, + ) + return result + + +class Vcpkg: + """Thin wrapper around a vcpkg installation.""" + + def __init__(self, vcpkg_root: Optional[pathlib.Path] = None): + self.exe = self._find_executable(vcpkg_root) + self.root = self._resolve_root(vcpkg_root) + self.installed_dir = self.root / "installed" + self.info_dir = self.installed_dir / "vcpkg" / "info" + + def _find_executable(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path: + # 1. Executable bundled inside the provided root. + if vcpkg_root: + for name in ("vcpkg.exe", "vcpkg"): + candidate = vcpkg_root / name + if candidate.exists(): + return candidate.resolve() + + # 2. Executable on PATH. + exe = shutil.which("vcpkg") + if exe: + return pathlib.Path(exe).resolve() + + # 3. Executable inside VCPKG_ROOT. + env_root = os.environ.get("VCPKG_ROOT") + if env_root: + for name in ("vcpkg.exe", "vcpkg"): + candidate = pathlib.Path(env_root) / name + if candidate.exists(): + return candidate.resolve() + + raise FileNotFoundError("vcpkg not found. Set VCPKG_ROOT or pass --vcpkg-root.") + + def _resolve_root(self, vcpkg_root: Optional[pathlib.Path]) -> pathlib.Path: + if vcpkg_root: + return vcpkg_root.resolve() + + env_root = os.environ.get("VCPKG_ROOT") + if env_root: + return pathlib.Path(env_root).resolve() + + # The executable normally lives at /vcpkg. + return self.exe.parent.resolve() + + def install(self, library: str, triplet: str) -> None: + """Install a library for the given triplet.""" + spec = f"{library}:{triplet}" + logger.info("vcpkg install %s", spec) + try: + run([str(self.exe), "install", spec]) + except subprocess.CalledProcessError as exc: + output = (exc.stdout or "") + (exc.stderr or "") + if "is only supported on" in output: + raise UnsupportedPlatformError(f"{spec} is not supported on this platform") from exc + raise + + def get_installed_version(self, library: str, triplet: str) -> str: + """Return the installed version string (e.g. '3.0.7#1').""" + result = run([str(self.exe), "list", f"{library}:{triplet}"]) + expected_prefix = f"{library}:{triplet}" + + for line in result.stdout.splitlines(): + line = line.strip() + if not line or line.startswith("The following packages are"): + continue + + parts = line.split() + if len(parts) < 2: + continue + + if parts[0] == expected_prefix: + return parts[1] + + raise BuildError(f"could not determine installed version for {library}:{triplet}") + + def find_package_libs(self, library: str, triplet: str) -> List[pathlib.Path]: + """Return static-library files (.lib/.a) owned by the given package.""" + # vcpkg records installed files in /installed/vcpkg/info/__.list + pattern = re.compile(re.escape(library) + r"_.*?_" + re.escape(triplet) + r"\.list$") + list_files = [] + if self.info_dir.exists(): + list_files = [p for p in self.info_dir.iterdir() if pattern.match(p.name)] + + if not list_files: + # Fallback: scan the whole lib directory. This is less precise but works + # when the .list file cannot be located. + logger.warning( + "could not find vcpkg info .list for %s:%s; falling back to lib-dir scan", + library, + triplet, + ) + return self._find_all_static_libs(triplet) + + lib_paths: List[pathlib.Path] = [] + for list_file in list_files: + for line in list_file.read_text().splitlines(): + line = line.strip() + if not line: + continue + if line.startswith(triplet + "/lib/"): + candidate = self.installed_dir / line + if candidate.suffix in (".lib", ".a"): + lib_paths.append(candidate) + + return sorted(set(lib_paths)) + + def _find_all_static_libs(self, triplet: str) -> List[pathlib.Path]: + lib_dir = self.installed_dir / triplet / "lib" + if not lib_dir.exists(): + return [] + return sorted(p for p in lib_dir.iterdir() if p.suffix in (".lib", ".a")) + + +class JHExtractor: + """Wrapper around the jh binary. Builds it from source if needed.""" + + def __init__( + self, + jh_path: Optional[pathlib.Path] = None, + lancelot_dir: Optional[pathlib.Path] = None, + ): + self.jh_path = self._resolve(jh_path, lancelot_dir) + + def _resolve( + self, + jh_path: Optional[pathlib.Path], + lancelot_dir: Optional[pathlib.Path], + ) -> pathlib.Path: + if jh_path: + path = pathlib.Path(jh_path).resolve() + if not path.exists(): + raise FileNotFoundError(f"jh binary not found: {path}") + return path + + env_path = os.environ.get("JH_PATH") + if env_path: + path = pathlib.Path(env_path).resolve() + if path.exists(): + return path + + if lancelot_dir: + return self._build(lancelot_dir) + + env_lancelot = os.environ.get("LANCELOT_DIR") + if env_lancelot: + return self._build(pathlib.Path(env_lancelot)) + + exe = shutil.which("jh") + if exe: + return pathlib.Path(exe).resolve() + + raise FileNotFoundError("jh not found. Provide --jh-path, --lancelot-dir, or set JH_PATH.") + + def _build(self, lancelot_dir: pathlib.Path) -> pathlib.Path: + logger.info("building jh from %s", lancelot_dir) + run( + ["cargo", "build", "--release", "-p", "lancelot-bin"], + cwd=lancelot_dir, + ) + exe = lancelot_dir / "target" / "release" / "jh" + if sys.platform == "win32": + exe = exe.with_suffix(".exe") + if not exe.exists(): + raise FileNotFoundError(f"jh binary not found after build: {exe}") + return exe.resolve() + + def extract( + self, + lib_path: pathlib.Path, + library: str, + version: str, + triplet: str, + compiler: str, + profile: str, + ) -> str: + """Run jh on a single static library and return its CSV output.""" + cmd = [ + str(self.jh_path), + triplet, + compiler, + library, + version, + profile, + str(lib_path), + ] + try: + result = run(cmd) + except subprocess.CalledProcessError as exc: + logger.error( + "jh failed for %s (%s): stdout=%r stderr=%r", + library, + lib_path.name, + exc.stdout, + exc.stderr, + ) + raise + return result.stdout + + +class Converter: + """Convert jh JSONL output into a gzip-compressed JSONL database.""" + + def __init__(self, emit_function_names: bool = True, deduplicate: bool = True): + self.emit_function_names = emit_function_names + self.deduplicate = deduplicate + + def parse( + self, + jh_text: str, + library: str, + version: str, + ) -> List[dict]: + """Parse jh JSONL into a list of entries (within-library dedup applied if enabled).""" + entries: List[dict] = [] + function_names: Set[str] = set() + explicit_function_names: Set[str] = set() + + for line in jh_text.splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + logger.warning("skipping malformed JSONL line: %s (%s)", line, exc) + continue + + file_path = row.get("path") + function_name = row.get("function") + feat_type = row.get("type") + value = row.get("value") + + if not function_name: + continue + + function_names.add(function_name) + + if feat_type == "string": + entries.append( + { + "string": value, + "library_name": library, + "library_version": version, + "file_path": file_path, + "function_name": function_name, + "line_number": None, + } + ) + elif feat_type == "function_name": + # Future-proof: a minimal extractor may emit function names explicitly. + entries.append( + { + "string": value, + "library_name": library, + "library_version": version, + "file_path": file_path, + "function_name": value, + "line_number": None, + } + ) + explicit_function_names.add(value) + + # Stock jh does not emit function_name rows, so derive them from the + # function column. Functions without any string/number/api features + # will be missed unless the extractor is patched to emit them. + if self.emit_function_names: + for fn in function_names - explicit_function_names: + entries.append( + { + "string": fn, + "library_name": library, + "library_version": version, + "file_path": None, + "function_name": fn, + "line_number": None, + } + ) + + if self.deduplicate: + # Match loader semantics: one metadata object per unique string. + seen: dict = {} + for entry in entries: + key = entry["string"] + if key not in seen: + seen[key] = entry + entries = list(seen.values()) + + return entries + + def write( + self, + entries: List[dict], + output_path: pathlib.Path, + ) -> dict: + """Write entries to a gzip-compressed JSONL file. Returns counts.""" + output_path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(output_path, "wt", encoding="utf-8") as f: + for entry in entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + num_string_entries = sum( + 1 for e in entries if e["function_name"] is not None and e["function_name"] != e["string"] + ) + num_function_name_entries = sum( + 1 for e in entries if e["function_name"] is not None and e["function_name"] == e["string"] + ) + + return { + "num_string_entries": num_string_entries, + "num_function_name_entries": num_function_name_entries, + "total_entries": len(entries), + } + + def convert( + self, + jh_text: str, + library: str, + version: str, + output_path: pathlib.Path, + ) -> dict: + """Convenience wrapper: parse jh JSONL and write JSONL.gz. Returns counts.""" + entries = self.parse(jh_text, library, version) + return self.write(entries, output_path) + + +def count_jsonl_rows(jh_text: str) -> dict: + """Count objects and unique functions referenced in jh JSONL output.""" + objects: Set[str] = set() + functions: Set[str] = set() + for line in jh_text.splitlines(): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + continue + file_path = row.get("path") + function_name = row.get("function") + if function_name: + objects.add(file_path) + functions.add(function_name) + return {"num_objects": len(objects), "num_functions": len(functions)} + + +def build_library( + library: str, + config: BuildConfig, + vcpkg: Vcpkg, + jh: JHExtractor, + converter: Converter, +) -> Tuple[LibraryMetrics, List[dict]]: + """Parse a single library's strings. Returns metrics and the deduped entries. + + The returned entries are not yet written to disk; the caller is responsible + for cross-library deduplication and final file emission. + """ + import time + + start = time.time() + metrics = LibraryMetrics( + library=library, + version="unknown", + triplet=config.triplet, + ) + entries: List[dict] = [] + + try: + vcpkg.install(library, config.triplet) + version = vcpkg.get_installed_version(library, config.triplet) + metrics.version = version + + lib_paths = vcpkg.find_package_libs(library, config.triplet) + if not lib_paths: + logger.info( + "%s: no static libraries found for %s:%s (likely header-only); skipping extraction", + library, + library, + config.triplet, + ) + else: + logger.info( + "%s: found %d static library file(s): %s", + library, + len(lib_paths), + ", ".join(str(p.name) for p in lib_paths), + ) + + all_jh_parts: List[str] = [] + for lib_path in lib_paths: + logger.info("%s: extracting strings from %s", library, lib_path.name) + jh_text = jh.extract( + lib_path, + library, + version, + config.triplet, + config.compiler, + config.profile, + ) + all_jh_parts.append(jh_text) + + combined_jh_text = "\n".join(all_jh_parts) + row_counts = count_jsonl_rows(combined_jh_text) + metrics.num_objects = row_counts["num_objects"] + metrics.num_functions = row_counts["num_functions"] + + entries = converter.parse(combined_jh_text, library, version) + metrics.num_raw_entries = len(entries) + except UnsupportedPlatformError as exc: + logger.warning("%s: skipping unsupported library (%s)", library, exc) + except Exception as exc: + logger.exception("%s: build failed", library) + metrics.error = f"{type(exc).__name__}: {exc}" + finally: + metrics.duration_seconds = time.time() - start + + return metrics, entries + + +def load_existing_entries(path: pathlib.Path) -> List[dict]: + """Load entries from an existing OSS database .jsonl.gz. + + Returns an empty list if the file is missing, empty, unreadable, or does + not contain entries in the expected schema. New keys are silently dropped; + missing keys are filled with None so the entries can be merged uniformly. + """ + if not path.exists(): + return [] + try: + raw = gzip.decompress(path.read_bytes()) + except (OSError, gzip.BadGzipFile, EOFError) as exc: + logger.warning("could not read existing database %s: %s", path, exc) + return [] + + entries: List[dict] = [] + for line in raw.split(b"\n"): + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + logger.warning("skipping malformed line in %s: %s", path, exc) + continue + if not isinstance(row, dict) or "string" not in row: + continue + entries.append( + { + "string": row.get("string"), + "library_name": row.get("library_name"), + "library_version": row.get("library_version"), + "file_path": row.get("file_path"), + "function_name": row.get("function_name"), + "line_number": row.get("line_number"), + "count": row.get("count", 1), + } + ) + return entries + + +def write_entries(entries: List[dict], output_path: pathlib.Path) -> None: + """Write entries to a gzip-compressed JSONL file.""" + output_path.parent.mkdir(parents=True, exist_ok=True) + with gzip.open(output_path, "wt", encoding="utf-8") as f: + for entry in entries: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + +def _entries_unchanged(existing: List[dict], new: List[dict]) -> bool: + """Return True if ``existing`` and ``new`` have the same string set and per-string count. + + Used to decide whether a preserved library's on-disk database needs to be + rewritten: a rewrite is only required when the string set or any + per-string count differs. + """ + existing_strings = {e["string"] for e in existing} + new_strings = {e["string"] for e in new} + if existing_strings != new_strings: + return False + existing_counts = {e["string"]: e.get("count", 1) for e in existing} + new_counts = {e["string"]: e["count"] for e in new} + return existing_counts == new_counts + + +def merge_entries( + new_entries: List[dict], + existing_entries: List[dict], + deduplicate: bool, +) -> List[dict]: + """Combine new and existing entries, with new taking precedence on conflict. + + When ``deduplicate`` is true the result contains at most one entry per + unique string value; otherwise the lists are concatenated as-is. + """ + if not new_entries: + return list(existing_entries) + if not existing_entries: + return list(new_entries) + if not deduplicate: + return list(existing_entries) + list(new_entries) + + seen: Dict[str, dict] = {} + # Iterate new first so that freshly-built entries win on string collisions, + # which is the right behavior when the underlying library version changes. + for entry in list(new_entries) + list(existing_entries): + key = entry["string"] + if key not in seen: + seen[key] = entry + return list(seen.values()) + + +def write_library_database( + metrics: LibraryMetrics, + entries: List[dict], + output_dir: pathlib.Path, + converter: Converter, +) -> LibraryMetrics: + """Write the per-library JSONL.gz and update metrics. Returns metrics.""" + output_path = output_dir / f"{metrics.library}.jsonl.gz" + + if not entries: + if output_path.exists(): + output_path.unlink() + logger.info("%s: removed empty database %s", metrics.library, output_path) + metrics.num_string_entries = 0 + metrics.num_function_name_entries = 0 + metrics.total_entries = 0 + return metrics + + counts = converter.write(entries, output_path) + metrics.num_string_entries = counts["num_string_entries"] + metrics.num_function_name_entries = counts["num_function_name_entries"] + metrics.total_entries = counts["total_entries"] + logger.info( + "%s: wrote %s (%d entries)", + metrics.library, + output_path, + metrics.total_entries, + ) + return metrics + + +def load_config(path: pathlib.Path) -> dict: + """Load build configuration from a JSON file.""" + data = json.loads(path.read_text()) + if not isinstance(data, dict): + raise ValueError(f"config file {path} must contain a JSON object") + return data + + +def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + # First pass: figure out if a config file was provided. + pre_parser = argparse.ArgumentParser(add_help=False) + pre_parser.add_argument("--config", type=pathlib.Path, default=None) + pre_args, _ = pre_parser.parse_known_args(argv) + + config: dict = {} + if pre_args.config: + config = load_config(pre_args.config) + elif "CONFIG" in os.environ: + config = load_config(pathlib.Path(os.environ["CONFIG"])) + + # Defaults are taken from (lowest to highest precedence): + # built-in constants < config file < environment variables < CLI args + defaults = { + "triplet": config.get("triplet"), + "compiler": config.get("compiler"), + "profile": config.get("profile"), + "libraries": config.get( + "libraries", + ), + } + + parser = argparse.ArgumentParser( + description="Build OSS string databases from vcpkg libraries.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + parents=[pre_parser], + ) + parser.set_defaults(**defaults) + parser.add_argument( + "--triplet", + help="vcpkg triplet", + ) + parser.add_argument( + "--compiler", + help="compiler label passed to jh", + ) + parser.add_argument( + "--profile", + help="build profile label passed to jh", + ) + parser.add_argument( + "--libraries", + nargs="+", + help="libraries to build", + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=pathlib.Path(__file__).parent, + help="directory for generated .jsonl.gz files and metrics", + ) + parser.add_argument( + "--vcpkg-root", + type=pathlib.Path, + default=os.environ.get("VCPKG_ROOT", None), + help="vcpkg installation root", + ) + parser.add_argument( + "--jh-path", + type=pathlib.Path, + default=os.environ.get("JH_PATH", None), + help="path to an existing jh binary", + ) + parser.add_argument( + "--lancelot-dir", + type=pathlib.Path, + default=os.environ.get("LANCELOT_DIR", None), + help="directory containing lancelot source; jh will be built if --jh-path is not given", + ) + parser.add_argument( + "--no-function-names", + action="store_true", + help="do not emit function-name-as-string entries", + ) + parser.add_argument( + "--no-deduplicate", + action="store_true", + help="emit one JSON object per CSV row instead of one per unique string", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help="continue building remaining libraries if one fails and exit successfully", + ) + parser.add_argument( + "--log-level", + default=os.environ.get("LOG_LEVEL", "INFO"), + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + help="logging level", + ) + return parser.parse_args(argv) + + +def main(argv: Optional[List[str]] = None) -> int: + args = parse_args(argv) + logging.getLogger().setLevel(args.log_level.upper()) + + config = BuildConfig( + triplet=args.triplet, + compiler=args.compiler, + profile=args.profile, + libraries=[lib.strip() for lib in args.libraries if lib.strip()], + output_dir=args.output_dir.resolve(), + vcpkg_root=args.vcpkg_root.resolve() if args.vcpkg_root else None, + jh_path=args.jh_path.resolve() if args.jh_path else None, + lancelot_dir=args.lancelot_dir.resolve() if args.lancelot_dir else None, + emit_function_names=not args.no_function_names, + deduplicate=not args.no_deduplicate, + continue_on_error=args.continue_on_error, + ) + + logger.info("configuration: %s", config) + + vcpkg = Vcpkg(config.vcpkg_root) + jh = JHExtractor(config.jh_path, config.lancelot_dir) + converter = Converter( + emit_function_names=config.emit_function_names, + deduplicate=config.deduplicate, + ) + + config.output_dir.mkdir(parents=True, exist_ok=True) + + metrics: List[LibraryMetrics] = [] + per_library_new: Dict[str, List[dict]] = {} + failed = False + for library in config.libraries: + metric, entries = build_library(library, config, vcpkg, jh, converter) + metrics.append(metric) + # Even on error, preserve any partial entries so cross-library dedup + # still sees them (and so partial results aren't lost). + per_library_new[library] = entries + if metric.error: + failed = True + if not config.continue_on_error: + break + + # Discover all existing .jsonl.gz databases in the output directory. These + # include libraries we are rebuilding (whose fresh entries will be merged + # in) and libraries we are leaving alone (which still participate in + # cross-library dedup and may need to be rewritten if shared strings change). + existing_files = sorted(config.output_dir.glob("*.jsonl.gz")) + metric_by_lib: Dict[str, LibraryMetrics] = {m.library: m for m in metrics} + + def _lib_name_from_path(p: pathlib.Path) -> str: + # p.name looks like ".jsonl.gz"; Path.stem would only strip ".gz". + suffix = ".jsonl.gz" + if p.name.endswith(suffix): + return p.name[: -len(suffix)] + return p.stem + + merged: Dict[str, List[dict]] = {} + for path in existing_files: + lib = _lib_name_from_path(path) + existing = load_existing_entries(path) + if lib in per_library_new: + merged[lib] = merge_entries(per_library_new[lib], existing, config.deduplicate) + elif existing: + merged[lib] = existing + + # Libraries being built for the first time (no pre-existing file). + for lib, new_entries in per_library_new.items(): + if lib not in merged: + merged[lib] = list(new_entries) + + # Count how many distinct libraries each string appears in. A string's + # count is the "found in N libraries" signal exposed to the user via + # OpenSourceString.count, so that strings unique to one library (count=1) + # can be distinguished from strings shared across many libraries + # (count>=2). Strings are NOT removed across libraries: when the same + # string is observed in both zlib and curl (e.g. zlib is vendored into + # curl), it stays in both databases with count=2. + count_by_string: Dict[str, int] = {} + for entries in merged.values(): + seen_in_lib: Set[str] = set() + for entry in entries: + s = entry["string"] + if s in seen_in_lib: + continue + seen_in_lib.add(s) + count_by_string[s] = count_by_string.get(s, 0) + 1 + + # Stamp the count on every entry that we will write so downstream + # consumers can see how many libraries each string appears in. + for entries in merged.values(): + for entry in entries: + entry["count"] = count_by_string[entry["string"]] + + # Write rebuilt libraries and update their metrics. + for metric in metrics: + if metric.error: + logger.warning("%s: skipping database write due to build error", metric.library) + continue + entries = merged.get(metric.library, []) + write_library_database(metric, entries, config.output_dir, converter) + + # Rewrite preserved libraries only if their string set or per-string + # count changed. (If we just rebuilt them, write_library_database + # already handled it above.) The count changes whenever a string is + # added to or removed from any library in the merged set, including + # new libraries joining the build. + rebuilt_libs = set(metric_by_lib.keys()) + preserved_metrics: List[dict] = [] + for lib in sorted(merged.keys()): + if lib in rebuilt_libs: + continue + entries = merged[lib] + output_path = config.output_dir / f"{lib}.jsonl.gz" + existing = load_existing_entries(output_path) + if existing and _entries_unchanged(existing, entries): + logger.debug("%s: unchanged, leaving %s alone", lib, output_path.name) + continue + if not entries: + output_path.unlink(missing_ok=True) + logger.info("%s: removed empty database %s", lib, output_path) + else: + write_entries(entries, output_path) + logger.info( + "%s: rewrote %s (%d entries)", + lib, + output_path, + len(entries), + ) + preserved_metrics.append( + { + "library": lib, + "version": entries[0]["library_version"] if entries else None, + "triplet": config.triplet, + "num_string_entries": sum( + 1 + for e in entries + if e.get("function_name") is not None and e.get("function_name") != e.get("string") + ), + "num_function_name_entries": sum( + 1 + for e in entries + if e.get("function_name") is not None and e.get("function_name") == e.get("string") + ), + "num_raw_entries": len(entries), + "total_entries": len(entries), + "duration_seconds": 0.0, + "error": None, + } + ) + + summary = { + "triplet": config.triplet, + "compiler": config.compiler, + "profile": config.profile, + "libraries": [m.as_dict() for m in metrics], + "preserved_libraries": preserved_metrics, + "successful": sum(1 for m in metrics if not m.error), + "failed": sum(1 for m in metrics if m.error), + } + + metrics_path = config.output_dir / "build_metrics.json" + metrics_path.write_text(json.dumps(summary, indent=2)) + logger.info("wrote metrics to %s", metrics_path) + + if failed: + logger.error("one or more libraries failed to build") + if config.continue_on_error: + return 0 + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_oss_db.py b/tests/test_qs_oss_db.py similarity index 100% rename from tests/test_oss_db.py rename to tests/test_qs_oss_db.py