diff --git a/.cargo/config.toml b/.cargo/config.toml index 06ca8c11f..e06f4b4ef 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,3 +2,31 @@ [alias] xtask = "run --package xtask --" +# The local mirror of CI. A green run here means a green run on the PR — the gate +# list is `run_local_ci_gates` in xtask/src/main.rs, kept alongside the workflows +# in .github/workflows/ so the two can be read against each other. +ci = "run --locked --release --package xtask -- run-local-ci-gates" +# Just the source-walking gates: no workspace build, so it is the fast pre-commit +# check. `cargo ci` is the full one. +gates = "run --locked --release --package xtask -- check-all-source-gates" + +# `--release` on both, because these gates are syn-parsing ~7k files and the +# profile dominates: 11s release vs 24s debug. That is the difference between a +# pre-commit check you run and one you skip. CI deliberately does NOT mirror this +# — its job already compiles xtask in debug for `cargo test -p xtask`, and adding +# a second profile there would cost more compile time than the 13s it saves. +# +# `--locked` because the workflows use it. Without it these aliases can resolve a +# different dependency graph than CI, or quietly rewrite Cargo.lock, and then go +# green on a build the PR will not reproduce — which is exactly the promise +# `cargo ci` exists to make. + +# Incremental compilation stays ON here, deliberately. It was briefly disabled +# after `target/debug/incremental` was found holding 419 GB, but measuring the +# thing that actually matters says otherwise: a one-line edit to +# streamlib-engine then `cargo check -p streamlib-engine` takes 3s incremental +# and 8s without, and one crate's incremental state is ~300 MB. The 419 GB was +# months of accumulation across crates, profiles and feature permutations, which +# `rm -rf target/debug/incremental` reclaims in seconds — a periodic prune, not a +# 2.7x slower edit loop. CI sets CARGO_INCREMENTAL=0 in the workflows instead, +# where it is unambiguously right: a runner never rebuilds after an edit. diff --git a/.github/workflows/check-boundaries.yml b/.github/workflows/check-boundaries.yml deleted file mode 100644 index 4aa38de1f..000000000 --- a/.github/workflows/check-boundaries.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Check Boundaries - -# Layer 6 of the regression-prevention defense for the Vulkan RHI capability -# split — see docs/architecture/subprocess-rhi-parity.md. `cargo xtask -# check-boundaries` enforces five boundary invariants by code-search across -# the zone dirs (`runtime/`, `sdk/`, `adapters/`, `tools/`, `vendor/`), -# `examples/`, and `packages/`: -# 1. No `ash` imports or Cargo deps (vulkanalia is the canonical Vulkan -# binding, per #252). -# 2. `use vulkanalia` (and Cargo.toml `vulkanalia` deps) confined to -# RHI / consumer-rhi / adapter / codec crates — anyone reaching for -# raw vulkanalia outside those crates is breaking the RHI boundary. -# 3. Cdylibs and adapter crates depend on `streamlib-consumer-rhi`, NOT -# the full `streamlib` crate. The FullAccess capability boundary is -# type-system enforced by the cdylib's dep graph excluding `streamlib`. -# 4. Privileged Vulkan calls (`vkAllocateMemory`, `vkGetMemoryFdKHR`, -# `vkCreateComputePipelines`) live only inside the RHI. -# 5. Every `vulkanalia` / `vulkanalia-sys` / `vulkanalia-vma` dep in a -# member crate inherits from `[workspace.dependencies]` (the vendored -# `vendor/tatolab-vulkanalia*` fork crates). A direct version spec or a -# direct `tatolab-vulkanalia*` dep can silently pull crates.io -# upstream or bypass the workspace rename and lose the VMA 3.3.0 -# patch. The vendored dirs themselves are the documented exception. -# -# The check is grep-shaped on purpose: sub-second on a clean runner, no -# Cargo build of the workspace needed. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-boundaries: - name: xtask check-boundaries - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-boundaries - run: cargo run --locked -q -p xtask -- check-boundaries - - # Drift trip-wire for the vendored vulkanalia trees - # (vendor/tatolab-vulkanalia*): any byte change vs. the recorded hashes - # fails — the guard against a workspace fmt sweep (or any stray edit) - # silently rewriting vendored sources. Deliberate re-vendors update - # the recorded hashes in the same commit; see - # docs/architecture/vendored-vulkanalia.md. - - name: cargo xtask check-vendored-vulkanalia - run: cargo run --locked -q -p xtask -- check-vendored-vulkanalia - - - name: cargo test -p xtask check_boundaries (fixture tests) - run: cargo test --locked -p xtask check_boundaries - - - name: cargo test -p xtask check_vendored_vulkanalia (fixture tests) - run: cargo test --locked -p xtask check_vendored_vulkanalia diff --git a/.github/workflows/check-device-wait-idle.yml b/.github/workflows/check-device-wait-idle.yml deleted file mode 100644 index 1a61be190..000000000 --- a/.github/workflows/check-device-wait-idle.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Check Device Wait Idle - -# CI gate for RHI device-idle discipline. -# `cargo xtask check-device-wait-idle` walks the engine sources and fails -# if any raw `device_wait_idle` call bypasses `HostVulkanDevice::wait_idle` -# — every full-device stall must route through the single RHI entrypoint. -# -# Grep-shaped on purpose: sub-second on a clean runner, no workspace build. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-device-wait-idle: - name: xtask check-device-wait-idle - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-device-wait-idle - run: cargo run --locked -q -p xtask -- check-device-wait-idle - - - name: cargo test -p xtask check_device_wait_idle (fixture tests) - run: cargo test --locked -p xtask check_device_wait_idle diff --git a/.github/workflows/check-no-escalate-in-lifecycle.yml b/.github/workflows/check-no-escalate-in-lifecycle.yml deleted file mode 100644 index a3ae5c962..000000000 --- a/.github/workflows/check-no-escalate-in-lifecycle.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Check No Escalate In Lifecycle - -# CI gate for the sandbox lifecycle contract. -# `cargo xtask check-no-escalate-in-lifecycle` walks the engine sources -# and fails if any `.escalate(...)` call appears inside a FullAccess -# lifecycle body (setup / teardown / start / stop and their `_inner` -# helpers), where escalate ops are not permitted. -# -# Grep-shaped on purpose: sub-second on a clean runner, no workspace build. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-no-escalate-in-lifecycle: - name: xtask check-no-escalate-in-lifecycle - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-no-escalate-in-lifecycle - run: cargo run --locked -q -p xtask -- check-no-escalate-in-lifecycle - - - name: cargo test -p xtask check_no_escalate_in_lifecycle (fixture tests) - run: cargo test --locked -p xtask check_no_escalate_in_lifecycle diff --git a/.github/workflows/check-no-in-process-placement.yml b/.github/workflows/check-no-in-process-placement.yml deleted file mode 100644 index c7fda4190..000000000 --- a/.github/workflows/check-no-in-process-placement.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Check No In-Process Placement - -# CI gate for the helper-process-placement-only ruling (owner 2026-08-04). -# `cargo xtask check-no-in-process-placement` walks the engine tree and -# `docs/` and fails on the vocabulary of the banned model — the patterns and -# the two escape hatches are enumerated in -# `xtask/src/check_no_in_process_placement.rs`. -# -# Markdown and Rust doc comments are in scope on purpose: the shipped -# violation announced itself in a `//!` line that three review rounds read -# past. See `docs/decisions/helper-process-placement-only.md` and -# `.claude/rules/placement.md`. -# -# Vocabulary, not behaviour — the behavioural proof that the parent never -# hosts a processor class is -# `sdk/streamlib-python-wheel/tests/test_helper_placement.py`. -# -# Grep-shaped on purpose: sub-second on a clean runner, no workspace build. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-no-in-process-placement: - name: xtask check-no-in-process-placement - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-no-in-process-placement - run: cargo run --locked -q -p xtask -- check-no-in-process-placement - - - name: cargo test -p xtask check_no_in_process_placement (fixture tests) - run: cargo test --locked -p xtask check_no_in_process_placement diff --git a/.github/workflows/check-no-inventory-submit.yml b/.github/workflows/check-no-inventory-submit.yml deleted file mode 100644 index 8eedb799c..000000000 --- a/.github/workflows/check-no-inventory-submit.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Check No inventory::submit!(FactoryRegistration) - -# CI gate for issue #793's all-dynamic registration rule. -# `cargo xtask check-no-inventory-submit` walks every workspace `.rs` -# file under the zone dirs (`runtime/`, `sdk/`, `adapters/`, `tools/`, -# `vendor/`), `packages/`, and `examples/` and fails if any -# non-`#[cfg(test)]` item reintroduces -# `inventory::submit!(FactoryRegistration { ... })`. The -# `#[processor]` macro no longer emits one, and reintroducing the -# pattern would bypass the dynamic-load model from the All-Dynamic -# Package Loading milestone (#20). -# -# `RuntimeInitHookRegistration` inventory submissions are a separate -# registration system and are NOT flagged — only `FactoryRegistration` -# is. Test code, doc strings, and `#[cfg(test)]`-gated items are -# exempt via Rust AST walking. -# -# Grep-shaped on purpose: sub-second on a clean runner, no workspace build. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-no-inventory-submit: - name: xtask check-no-inventory-submit - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-no-inventory-submit - run: cargo run --locked -q -p xtask -- check-no-inventory-submit - - - name: cargo test -p xtask check_no_inventory_submit (fixture tests) - run: cargo test --locked -p xtask check_no_inventory_submit diff --git a/.github/workflows/check-no-unbounded-cstr-from-ptr.yml b/.github/workflows/check-no-unbounded-cstr-from-ptr.yml deleted file mode 100644 index 07f0af247..000000000 --- a/.github/workflows/check-no-unbounded-cstr-from-ptr.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Check No Unbounded CStr From Ptr - -# CI gate for borrow-checked C strings in the Vulkan RHI. -# `cargo xtask check-no-unbounded-cstr-from-ptr` walks the RHI trees and fails -# on any `CStr::from_ptr(.as_ptr())` — that spelling returns an -# unbounded lifetime, so the borrow outlives the storage it points into -# (the #1846 use-after-free). `vk::StringArray::as_cstr` is the drop-in. -# -# Grep-shaped on purpose: sub-second on a clean runner, no workspace build. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-check-no-unbounded-cstr-from-ptr: - name: xtask check-no-unbounded-cstr-from-ptr - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask check-no-unbounded-cstr-from-ptr - run: cargo run --locked -q -p xtask -- check-no-unbounded-cstr-from-ptr - - - name: cargo test -p xtask check_no_unbounded_cstr_from_ptr (fixture tests) - run: cargo test --locked -p xtask check_no_unbounded_cstr_from_ptr diff --git a/.github/workflows/check-ship-change-removed-gate.yml b/.github/workflows/check-ship-change-removed-gate.yml deleted file mode 100644 index d4eb3d427..000000000 --- a/.github/workflows/check-ship-change-removed-gate.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Check Ship-Change Removed Gate - -# Unit tests for `.claude/scripts/ship-change-removed-gate.sh` — the only mechanism -# that proves a change's REMOVED inventory actually left the tree before the change -# may archive. -# -# The gate shipped without tests and was green on 25 of 36 bullets whose artifacts -# were all still on disk. Every way a bullet can pass vacuously now has a case in -# `.claude/scripts/tests/ship-change-removed-gate.test.sh`; this job is what keeps -# them from rotting back. -# -# Only the gate's own tests run here. Active change files legitimately fail the gate -# — their removals have not happened yet — so CI never runs it against them. -# -# bash + git, no toolchain, no network: sub-second on a clean runner. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -jobs: - ship-change-removed-gate-tests: - name: ship-change-removed-gate tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - # The job runs bash tests over throwaway repos it creates itself — no - # authenticated git operation, so the checkout token has no business in - # the local git config. - persist-credentials: false - - - name: bash .claude/scripts/tests/ship-change-removed-gate.test.sh - run: bash .claude/scripts/tests/ship-change-removed-gate.test.sh diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml deleted file mode 100644 index c612b091b..000000000 --- a/.github/workflows/license-check.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: License Header Check - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - check-headers: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Check Rust files for copyright header - run: | - missing_files=() - while IFS= read -r -d '' file; do - # Shebang-aware: a `#!` interpreter line may legitimately occupy - # line 1; the copyright header then sits on line 2. - if head -1 "$file" | grep -q '^#!'; then - check_line=2 - else - check_line=1 - fi - if ! sed -n "${check_line}p" "$file" | grep -q "// Copyright (c) 2025 Jonathan Fontanez"; then - missing_files+=("$file") - fi - # The three vendored vulkanalia fork dirs are Apache-2.0 verbatim - # copies and deliberately carry NO BUSL headers — see - # docs/architecture/vendored-vulkanalia.md and CLAUDE.md's - # licensing exception. Exact-dir exclusions (a future - # vendor/tatolab-vulkanalia-extras/ crate would NOT be excluded). - done < <(find runtime sdk adapters vendor examples -name "*.rs" \ - -not -path "vendor/tatolab-vulkanalia/*" \ - -not -path "vendor/tatolab-vulkanalia-sys/*" \ - -not -path "vendor/tatolab-vulkanalia-vma/*" \ - -print0) - - if [ ${#missing_files[@]} -ne 0 ]; then - echo "❌ The following files are missing the required copyright header:" - echo "" - for file in "${missing_files[@]}"; do - echo " - $file" - done - echo "" - echo "Required header (first two lines of file):" - echo " // Copyright (c) 2025 Jonathan Fontanez" - echo " // SPDX-License-Identifier: BUSL-1.1" - exit 1 - fi - - echo "✅ All Rust files have the required copyright header" - - - name: Check Python files for copyright header - run: | - missing_files=() - while IFS= read -r -d '' file; do - # Shebang-aware: executable scripts start with `#!`; the copyright - # header then sits on line 2 (any `#!`-prefixed Python script). - if head -1 "$file" | grep -q '^#!'; then - check_line=2 - else - check_line=1 - fi - if ! sed -n "${check_line}p" "$file" | grep -q "# Copyright (c) 2025 Jonathan Fontanez"; then - missing_files+=("$file") - fi - done < <(find . -name "*.py" -not -path "./node_modules/*" -not -path "./.venv/*" -not -path "./target/*" -print0 2>/dev/null || true) - - if [ ${#missing_files[@]} -ne 0 ]; then - echo "❌ The following Python files are missing the required copyright header:" - echo "" - for file in "${missing_files[@]}"; do - echo " - $file" - done - echo "" - echo "Required header (first two lines of file):" - echo " # Copyright (c) 2025 Jonathan Fontanez" - echo " # SPDX-License-Identifier: BUSL-1.1" - exit 1 - fi - - echo "✅ All Python files have the required copyright header (or none exist)" diff --git a/.github/workflows/lint-logging.yml b/.github/workflows/lint-logging.yml deleted file mode 100644 index 67be9c7fb..000000000 --- a/.github/workflows/lint-logging.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Lint Logging - -# Enforces the unified logging pathway — see docs/logging.md. -# `cargo xtask lint-logging` bans the banned-macro/function set across all three -# surface areas without compiling the workspace: -# - Rust: syn-based AST walk over opt-in workspace crates (under `runtime/`, -# `sdk/`, `adapters/`, `tools/`, `vendor/`), honors -# `#[allow(clippy::disallowed_macros)]`, `#[cfg(test)]`, and platform cfg -# gates exactly as clippy would on `ubuntu-latest`. -# - Python: banned `print()`/`sys.stdout`/`logging.basicConfig` in -# sdk/streamlib-python and sdk/streamlib-python-wheel/python. -# -# Rationale for not using `cargo clippy --workspace`: it transitively compiles -# `libs/vulkan-video`, whose build.rs requires `glslc`. Keeping the lockout -# compile-free lets this workflow run in seconds on a clean runner. - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - CARGO_TERM_COLOR: always - -jobs: - xtask-lint-logging: - name: xtask lint-logging (Rust + Python) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-xtask-${{ hashFiles('xtask/Cargo.toml', 'xtask/Cargo.lock', 'Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-xtask- - - - name: cargo xtask lint-logging - run: cargo run --locked -q -p xtask -- lint-logging - - - name: cargo test -p xtask lint_logging (fixture tests) - run: cargo test --locked -p xtask lint_logging diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index a67c6545e..6d9b92b55 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -4,6 +4,10 @@ on: pull_request: types: [opened, edited, synchronize, reopened] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + permissions: pull-requests: read diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml index 5d587b427..9bc4796c1 100644 --- a/.github/workflows/python-wheel.yml +++ b/.github/workflows/python-wheel.yml @@ -3,8 +3,32 @@ name: Python Wheel on: push: branches: [main] + # Prose cannot break a wheel build or a type check, and this is the slowest + # job in the repo. `main` carries no required status checks, so a skipped run + # blocks nothing. `.github/**` is deliberately absent: a workflow edit must + # run the workflow. + paths-ignore: + - 'docs/**' + - '**.md' + - '.claude/**' + - 'LICENSE' + - 'LICENSES/**' pull_request: branches: [main] + paths-ignore: + - 'docs/**' + - '**.md' + - '.claude/**' + - 'LICENSE' + - 'LICENSES/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 jobs: wheel-interpreter-lifecycle: @@ -18,6 +42,10 @@ jobs: steps: - uses: actions/checkout@v4 + with: + # Compiles and runs PR-authored code; a persisted checkout token + # would be readable by it. No workspace member has a git dependency. + persist-credentials: false - name: Install system dependencies # Same minimal engine build set as test.yml: the engine's build.rs @@ -36,17 +64,15 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache Cargo - continue-on-error: true - uses: actions/cache@v4 + # Was a raw `actions/cache` over `target/`. Three near-identical 1.97 GB + # entries existed at once — one for main and one for each open PR — because + # Actions caches are ref-scoped and a PR's save is unreadable by any other + # PR. `save-if` on main means PRs restore and save nothing. + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-wheel-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-wheel- + shared-key: wheel + save-if: ${{ github.ref == 'refs/heads/main' }} # Scoped to the signals module rather than the whole engine lib suite, # which is a tracked follow-up (see test.yml) pending a parallel-run @@ -122,6 +148,10 @@ jobs: # correct hovers for anyone writing a processor. steps: - uses: actions/checkout@v4 + with: + # Compiles and runs PR-authored code; a persisted checkout token + # would be readable by it. No workspace member has a git dependency. + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@v5 diff --git a/.github/workflows/repo-gates.yml b/.github/workflows/repo-gates.yml new file mode 100644 index 000000000..c5a4022d8 --- /dev/null +++ b/.github/workflows/repo-gates.yml @@ -0,0 +1,48 @@ +name: Repo Gates + +# The toolchain-free gates: bash and git only, no Rust, no Python, no network. +# Both ran as their own workflow; together they finish in seconds, so they share +# one runner rather than two. +# +# - License headers: every Rust and Python file carries the BUSL header. The +# implementation is `scripts/check-license-headers.sh` rather than inline YAML +# so that `cargo xtask run-local-ci-gates` runs the identical check — inline +# workflow bash is, by construction, something no one can run before pushing. +# - ship-change-removed-gate: unit tests for the only mechanism that proves a +# change's REMOVED inventory actually left the tree before the change may +# archive. The gate shipped without tests and was green on 25 of 36 bullets +# whose artifacts were all still on disk. Only the gate's own tests run here — +# active change files legitimately fail the gate, since their removals have +# not happened yet, so CI never runs it against them. + +on: + pull_request: + branches: [main] + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + repo-gates: + name: License headers + ship-change removed gate + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + # The ship-change gate tests run over throwaway repos they create + # themselves — no authenticated git operation, so the checkout token + # has no business in the local git config. + persist-credentials: false + # `check-license-headers.sh` discovers files with `git ls-files`, which + # needs the work tree but not its history. + fetch-depth: 1 + + - name: License headers (Rust + Python) + run: bash scripts/check-license-headers.sh + + - name: ship-change-removed-gate tests + run: bash .claude/scripts/tests/ship-change-removed-gate.test.sh diff --git a/.github/workflows/schemas.yml b/.github/workflows/schemas.yml index 9be7db5be..49547f293 100644 --- a/.github/workflows/schemas.yml +++ b/.github/workflows/schemas.yml @@ -8,6 +8,7 @@ on: env: CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 jobs: generate-and-publish: @@ -43,16 +44,14 @@ jobs: registry-url: 'https://npm.pkg.github.com' scope: '@tatolab' - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 + # Dispatch-only, so it restores whatever main last saved rather than + # minting an entry of its own — the repo cache was over GitHub's 10 GB cap + # and every save evicted a live entry. + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-schemas-${{ hashFiles('**/Cargo.toml') }} + shared-key: wheel + save-if: false - name: Get version from Cargo.toml id: version diff --git a/.github/workflows/source-gates.yml b/.github/workflows/source-gates.yml new file mode 100644 index 000000000..e13c2eafc --- /dev/null +++ b/.github/workflows/source-gates.yml @@ -0,0 +1,84 @@ +name: Source Gates + +# Every source-walking gate, in one job, in one process. +# +# These ran as seven separate workflows. Each paid ~26s compiling the xtask +# binary and ~13s compiling its test binary to do a few seconds of file walking, +# and all seven shared one cache key — so they raced, and six of every seven +# cache saves were discarded. One job compiles xtask once and runs all eight +# gates plus the whole fixture suite. +# +# `check-all-source-gates` runs every gate before reporting, so one job still +# surfaces every breakage at once. The per-gate subcommands remain for narrowing +# a failure down locally; the gate list itself lives in `xtask/src/main.rs` +# (`ALL_SOURCE_WALKING_GATES`), and each gate's contract is documented on its +# `Commands` variant there. +# +# Two things worth keeping in view: +# - lint-logging deliberately does not shell out to `cargo clippy --workspace`. +# That would transitively compile `libs/vulkan-video`, whose build.rs needs +# `glslc`. A syn-based AST walk keeps this job compile-free apart from xtask +# itself, and honors `#[allow(clippy::disallowed_macros)]`, `#[cfg(test)]` and +# platform cfg gates exactly as clippy would on ubuntu-latest. +# - check-no-in-process-placement gates *vocabulary*, not behaviour. The +# behavioural proof that the parent never hosts a processor class is +# `sdk/streamlib-python-wheel/tests/test_helper_placement.py`. Markdown and +# Rust doc comments are in scope on purpose: the shipped violation announced +# itself in a `//!` line that three review rounds read past. +# +# No path filter. These gates scan `docs/` and Markdown as well as source, so a +# docs-only change is exactly when some of them earn their keep. + +on: + pull_request: + branches: [main] + push: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + # CI never rebuilds after an edit, so incremental artifacts are pure cache + # weight — locally they had grown to 411 GB. + CARGO_INCREMENTAL: 0 + +jobs: + source-gates: + name: xtask source gates + fixture tests + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + # This job compiles and runs PR-authored code (`cargo run`, `cargo + # test`, build scripts, proc macros). A persisted checkout token sits + # in `.git/config` where any of that can read it, and no workspace + # member has a git dependency, so nothing here needs credentials. + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + # Replaces a raw `actions/cache` over `target/`. That cached every + # intermediate artifact including the workspace's own crates, which are + # invalid on the next commit anyway; rust-cache keeps dependency artifacts + # and drops the rest. + # + # `save-if` on main is the fix for cache eviction. Actions caches are + # ref-scoped: a PR's save is unreadable by every other PR, so each PR was + # minting its own multi-GB entry and evicting main's. PRs now restore from + # main and save nothing. + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 + with: + shared-key: xtask + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: cargo xtask check-all-source-gates + run: cargo run --locked -q -p xtask -- check-all-source-gates + + - name: cargo test -p xtask (all gate fixture tests) + run: cargo test --locked -p xtask diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9cfb1a3ca..8196e4270 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,19 +3,44 @@ name: Test on: pull_request: branches: [main] + # Prose changes cannot break a Rust unit test, and this job is ~2.5 minutes. + # `main` carries no required status checks, so a skipped run blocks nothing. + # `.github/**` is deliberately absent: a workflow edit must run the workflow. + paths-ignore: + - 'docs/**' + - '**.md' + - '.claude/**' + - 'LICENSE' + - 'LICENSES/**' push: branches: [main] + paths-ignore: + - 'docs/**' + - '**.md' + - '.claude/**' + - 'LICENSE' + - 'LICENSES/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 jobs: rust-tests-linux: name: Rust Tests (Linux) runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v4 + with: + # Compiles and runs PR-authored code; a persisted checkout token would + # be readable by it. No workspace member has a git dependency. + persist-credentials: false - name: Install system dependencies # `-p streamlib` pulls streamlib-engine, whose build.rs compiles GLSL @@ -33,18 +58,19 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable - - name: Cache Cargo - # A cache-service flake must never fail the job; a miss just means a cold build. - continue-on-error: true - uses: actions/cache@v4 + # Was a raw `actions/cache` over `target/`, which had grown to a 3.72 GB + # entry taking 66s to restore — longer than the 53s of tests it existed to + # accelerate, so it was a net loss. rust-cache keeps dependency artifacts + # and drops the workspace's own, which are invalid next commit anyway. + # + # `save-if` on main stops PRs minting their own ref-scoped copies. Repo + # cache had reached 11.76 GB against GitHub's 10 GB cap, so every save was + # evicting something — including the entry the next run needed. + - name: Cache Rust dependencies + uses: Swatinem/rust-cache@v2 with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-test- + shared-key: rust-tests + save-if: ${{ github.ref == 'refs/heads/main' }} # Unit-test gate. Runs the facade + macros lib tests plus the # `#[processor]` emission locks. streamlib-macros carries the @@ -57,4 +83,3 @@ jobs: run: | cargo test --locked -p streamlib -p streamlib-macros --lib cargo test --locked -p streamlib-engine --test attribute_macro_test - diff --git a/scripts/check-license-headers.sh b/scripts/check-license-headers.sh new file mode 100755 index 000000000..edea45f10 --- /dev/null +++ b/scripts/check-license-headers.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Copyright (c) 2025 Jonathan Fontanez +# SPDX-License-Identifier: BUSL-1.1 +# +# Every Rust and Python source file carries the BUSL header. Run from the +# workspace root, by `.github/workflows/repo-gates.yml` and by +# `cargo xtask run-local-ci-gates` alike — one implementation, so a green local +# run means a green CI run. +# +# File discovery is `git ls-files`, not `find`. CI walks a clean checkout, so +# "the files in the repo" is the semantics the gate has always meant; a `find` +# reproduces it on the runner and then reports thousands of false positives on a +# developer machine, where venvs, uv caches and build output sit in the tree. +# `--others --exclude-standard` adds new-but-unstaged files, so a header missing +# from a file you just created fails here rather than on the PR. + +set -uo pipefail + +list_repo_files() { + git ls-files -z --cached --others --exclude-standard -- "$@" +} + +# A `#!` interpreter line may legitimately occupy line 1; the copyright header +# then sits on line 2. +# +# Both lines are checked, and as whole lines. Checking only the copyright line — +# which is what this gate did for its whole life before 2026-08-12 — passes a +# file that carries the copyright but no SPDX identifier, and SPDX is the half a +# licence scanner actually reads. +report_files_missing_header() { + local expected_header="$1" + local expected_spdx="${expected_header%%Copyright*}SPDX-License-Identifier: BUSL-1.1" + local language="$2" + shift 2 + + local missing_files=() + local file + local header_line_number + while IFS= read -r -d '' file; do + if head -1 "$file" | grep -q '^#!'; then + header_line_number=2 + else + header_line_number=1 + fi + if ! sed -n "${header_line_number}p" "$file" | grep -qxF "$expected_header" || + ! sed -n "$((header_line_number + 1))p" "$file" | grep -qxF "$expected_spdx"; then + missing_files+=("$file") + fi + done < <(list_repo_files "$@") + + if [ ${#missing_files[@]} -ne 0 ]; then + echo "❌ ${#missing_files[@]} $language file(s) are missing the required copyright header:" + echo "" + printf ' - %s\n' "${missing_files[@]}" + echo "" + echo "Required header (lines 1-2, or lines 2-3 after a shebang):" + echo " $expected_header" + echo " $expected_spdx" + return 1 + fi + + echo "✅ All $language files carry the required copyright header" + return 0 +} + +failed_language_checks=() + +# Every Rust file in the repo, not an enumerated set of zone dirs. The old list +# named runtime/ sdk/ adapters/ vendor/ examples/, which silently exempted +# `xtask/` and `tools/` — and `xtask/src/check_no_inventory_submit.rs` had been +# sitting there with a `2026` copyright line the rule does not permit. +# +# The three vendored vulkanalia fork dirs are Apache-2.0 verbatim copies and +# deliberately carry NO BUSL headers — see +# docs/architecture/vendored-vulkanalia.md and CLAUDE.md's licensing exception. +# Exact-dir exclusions (a future vendor/tatolab-vulkanalia-extras/ crate would +# NOT be excluded). +report_files_missing_header \ + "// Copyright (c) 2025 Jonathan Fontanez" Rust \ + '*.rs' \ + ':(exclude)vendor/tatolab-vulkanalia/*' \ + ':(exclude)vendor/tatolab-vulkanalia-sys/*' \ + ':(exclude)vendor/tatolab-vulkanalia-vma/*' || + failed_language_checks+=("Rust") + +report_files_missing_header \ + "# Copyright (c) 2025 Jonathan Fontanez" Python \ + '*.py' || + failed_language_checks+=("Python") + +if [ ${#failed_language_checks[@]} -ne 0 ]; then + echo "" + echo "❌ License header check failed for: ${failed_language_checks[*]}" + exit 1 +fi diff --git a/xtask/src/check_no_inventory_submit.rs b/xtask/src/check_no_inventory_submit.rs index cc8167f6d..9b40dfbae 100644 --- a/xtask/src/check_no_inventory_submit.rs +++ b/xtask/src/check_no_inventory_submit.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2026 Jonathan Fontanez +// Copyright (c) 2025 Jonathan Fontanez // SPDX-License-Identifier: BUSL-1.1 //! CI gate enforcing the all-dynamic registration rule for processor diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 5e8a6eb79..703f272be 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; pub mod check_boundaries; pub mod check_device_wait_idle; @@ -44,6 +44,158 @@ pub fn ensure_source_walking_gate_read_source( Ok(()) } +/// Every source-walking gate, paired with the subcommand name that runs it alone. +/// +/// Each gate reads the tree and reports; none builds the workspace. That is what +/// lets one process run all eight in well under a second, and why CI runs them as +/// a single job rather than one runner per gate. +const ALL_SOURCE_WALKING_GATES: &[(&str, fn(&Path) -> Result<()>)] = &[ + ("lint-logging", lint_logging::run), + ("check-boundaries", check_boundaries::run), + ("check-vendored-vulkanalia", check_vendored_vulkanalia::run), + ( + "check-no-in-process-placement", + check_no_in_process_placement::run, + ), + ("check-no-inventory-submit", check_no_inventory_submit::run), + ( + "check-no-escalate-in-lifecycle", + check_no_escalate_in_lifecycle::run, + ), + ("check-device-wait-idle", check_device_wait_idle::run), + ( + "check-no-unbounded-cstr-from-ptr", + check_no_unbounded_cstr_from_ptr::run, + ), +]; + +/// Run every source-walking gate, reporting all failures rather than the first. +/// +/// A gate that bails on first failure hides the rest behind a re-run, which is the +/// one thing a consolidated job must not reintroduce: eight separate jobs at least +/// told you about eight separate breakages at once. +fn run_all_source_walking_gates(workspace_root: &Path) -> Result<()> { + let mut failed_gate_names: Vec<&str> = Vec::new(); + + for (gate_name, run_gate) in ALL_SOURCE_WALKING_GATES { + match run_gate(workspace_root) { + Ok(()) => tracing::info!("PASS {gate_name}"), + Err(gate_failure) => { + tracing::error!("FAIL {gate_name}: {gate_failure:#}"); + failed_gate_names.push(gate_name); + } + } + } + + anyhow::ensure!( + failed_gate_names.is_empty(), + "{} of {} source-walking gates failed: {}", + failed_gate_names.len(), + ALL_SOURCE_WALKING_GATES.len(), + failed_gate_names.join(", ") + ); + + tracing::info!( + "all {} source-walking gates passed", + ALL_SOURCE_WALKING_GATES.len() + ); + Ok(()) +} + +/// Run one command from the workspace root, failing on a non-zero exit status. +fn run_local_ci_gate_command( + workspace_root: &Path, + gate_name: &str, + program: &str, + arguments: &[&str], +) -> Result<()> { + let exit_status = std::process::Command::new(program) + .args(arguments) + .current_dir(workspace_root) + .status() + .with_context(|| format!("failed to spawn `{program}` for {gate_name}"))?; + + anyhow::ensure!(exit_status.success(), "{gate_name} failed ({exit_status})"); + Ok(()) +} + +/// Run the gates CI runs, in the order CI runs them, reporting every failure. +/// +/// The point is that a green run here means a green run on the PR. Any gate added +/// to CI without being added here breaks that promise, so the two lists are meant +/// to be read side by side against `.github/workflows/`. +fn run_local_ci_gates(workspace_root: &Path) -> Result<()> { + let mut failed_gate_names: Vec<&str> = Vec::new(); + + if let Err(gate_failure) = run_all_source_walking_gates(workspace_root) { + tracing::error!("{gate_failure:#}"); + failed_gate_names.push("source-walking gates"); + } + + let shelled_out_gates: &[(&str, &str, &[&str])] = &[ + ( + "license headers", + "bash", + &["scripts/check-license-headers.sh"], + ), + ( + "ship-change removed gate tests", + "bash", + &[".claude/scripts/tests/ship-change-removed-gate.test.sh"], + ), + ( + "xtask gate fixture tests", + "cargo", + &["test", "--locked", "-p", "xtask"], + ), + ( + "SDK + macros unit tests", + "cargo", + &[ + "test", + "--locked", + "-p", + "streamlib", + "-p", + "streamlib-macros", + "--lib", + ], + ), + ( + "processor-macro emission locks", + "cargo", + &[ + "test", + "--locked", + "-p", + "streamlib-engine", + "--test", + "attribute_macro_test", + ], + ), + ]; + + for (gate_name, program, arguments) in shelled_out_gates { + tracing::info!("running {gate_name}"); + if let Err(gate_failure) = + run_local_ci_gate_command(workspace_root, gate_name, program, arguments) + { + tracing::error!("{gate_failure:#}"); + failed_gate_names.push(gate_name); + } + } + + anyhow::ensure!( + failed_gate_names.is_empty(), + "{} local CI gate(s) failed: {}", + failed_gate_names.len(), + failed_gate_names.join(", ") + ); + + tracing::info!("all local CI gates passed"); + Ok(()) +} + #[derive(Parser)] #[command(name = "xtask")] #[command(about = "StreamLib development tasks")] @@ -119,6 +271,15 @@ enum Commands { /// recorded hashes in the same commit per /// `docs/architecture/vendored-vulkanalia.md`. CheckVendoredVulkanalia, + + /// Run every source-walking gate in one process and report all failures. + /// This is what CI's `source-gates` job runs; the per-gate subcommands stay + /// for narrowing down a failure locally. + CheckAllSourceGates, + + /// Run the gates CI runs, so a green run here predicts a green PR. Builds + /// the workspace, so it is slower than `check-all-source-gates` alone. + RunLocalCiGates, } fn main() -> Result<()> { @@ -148,6 +309,8 @@ fn main() -> Result<()> { check_no_unbounded_cstr_from_ptr::run(&workspace_root()?)? } Commands::CheckVendoredVulkanalia => check_vendored_vulkanalia::run(&workspace_root()?)?, + Commands::CheckAllSourceGates => run_all_source_walking_gates(&workspace_root()?)?, + Commands::RunLocalCiGates => run_local_ci_gates(&workspace_root()?)?, } Ok(())