diff --git a/cpp/.clang-tidy b/cpp/.clang-tidy new file mode 100644 index 000000000..39555fcab --- /dev/null +++ b/cpp/.clang-tidy @@ -0,0 +1,4 @@ +Checks: "-*,clang-analyzer-core.*,clang-analyzer-cplusplus.*,clang-analyzer-unix.Malloc,clang-analyzer-unix.MismatchedDeallocator,bugprone-dangling-handle,bugprone-use-after-move" +WarningsAsErrors: "*" +InheritParentConfig: false +HeaderFilterRegex: ".*" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d693ee9f8..a8e7c7897 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -47,6 +47,16 @@ install(TARGETS uos_cpp_pilot RUNTIME DESTINATION bin) enable_testing() add_executable(protocol_tests tests/protocol_tests.cpp) target_link_libraries(protocol_tests PRIVATE pilot_core) +option(UOS_ANALYSIS_CONTROLS "Expose analysis-only fixture compilation commands" OFF) +if(UOS_ANALYSIS_CONTROLS) + add_library(analysis_controls OBJECT EXCLUDE_FROM_ALL tests/analysis/raw_defects.cpp tests/analysis/clean_ownership.cpp tests/analysis/clean_cancel.cpp) + target_link_libraries(analysis_controls PRIVATE pilot_core) + target_include_directories(analysis_controls SYSTEM PRIVATE tests/analysis/fixture_sdk) + add_custom_target(cpp-static-analysis + COMMAND /usr/bin/python3 "${PROJECT_SOURCE_DIR}/tools/analyze.py" "$ENV{UOS_CPP_TOOLCHAIN_DIR}" pi-release + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}/.." + VERBATIM) +endif() if(UOS_SANITIZE) target_compile_options(protocol_tests PRIVATE -fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer) target_link_options(protocol_tests PRIVATE -fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer) diff --git a/cpp/CMakePresets.json b/cpp/CMakePresets.json index 1aad42672..3c90af117 100644 --- a/cpp/CMakePresets.json +++ b/cpp/CMakePresets.json @@ -14,6 +14,7 @@ "CMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY": "OFF", "CMAKE_TOOLCHAIN_FILE": "$env{UOS_CPP_TOOLCHAIN_DIR}/probe/pi-arm64.cmake", "CMAKE_BUILD_TYPE": "Release", + "UOS_ANALYSIS_CONTROLS": "ON", "CMAKE_CXX_FLAGS_RELEASE": "-O2 -DNDEBUG -g1", "CMAKE_CXX_FLAGS": "-B$env{UOS_CPP_TOOLCHAIN_DIR}/host-tools/bin/", "CMAKE_AR": "$env{UOS_CPP_TOOLCHAIN_DIR}/host-tools/bin/ar", diff --git a/cpp/README.md b/cpp/README.md index 856aacc54..597763638 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -20,6 +20,8 @@ python3 cpp/tools/build.py /absolute/qualified-sdk native-asan test python3 cpp/tools/build.py /absolute/qualified-sdk pi-release tidy ``` +`pi-release complete` runs configure, build, all six CTests and the mandatory static-analysis gate. `tidy` runs that same gate separately; the always-run CMake target is `cpp-static-analysis`. The gate verifies the Pi compilation database, analyzes all twelve production translation units plus `protocol_tests.cpp`, and checks three analysis-only controls. See [static-analysis.md](docs/static-analysis.md) for the selected checks, receipts and limitations. + The driver creates one reusable directory per preset under `.data/cpp-build`, a task-local CMake executable view, disk-backed compiler scratch, and full command/resource logs under `.data/cpp-evidence`. Its view executes the same qualified SDK binaries through their pinned loader with `--argv0`, so generated archive, regeneration, test and install commands use that same loader. The SDK itself stays read-only; no global library path changes are required. Build jobs remain1. The first512MiB compiler virtual-memory limit was measured insufficient for the transport translation unit. The owner authorized768MiB for the compiler and bounded build cgroup; this is distinct from the candidate's provisional96MiB service-memory acceptance target. The one-second build sampler reports a lower-bound disk peak, enforces the additional768MiB/aggregate2GB storage envelope and refuses filesystem free space below2GiB. Native ASan runtime requires its shadow address space and does not inherit the compiler VM cap. diff --git a/cpp/clang14-beast-read-baseline.json b/cpp/clang14-beast-read-baseline.json new file mode 100644 index 000000000..f7db57dcb --- /dev/null +++ b/cpp/clang14-beast-read-baseline.json @@ -0,0 +1,139 @@ +{ + "schema": "uos-clang14-beast-read-v1", + "source": "cpp/src/session.cpp", + "sdk_sha256": { + "root/lib/ld-linux-aarch64.so.1": "7fa5037b2ea1643fd2b8b7f3f3c1d9d546271011b4385aeea3e85fed0d5dcadd", + "root/usr/bin/clang-tidy-14": "c66bd47837a3621500d26b3ec4636890eaf6c85f0afefd5b3d8f570eecdc8e90", + "root/usr/include/boost/beast/http/impl/read.hpp": "dcace292f59d5e5d6ca45930fb02eeea4418f940c91af52de28de2fa6f53a6ce", + "root/usr/include/boost/asio/coroutine.hpp": "aa981b4602747d1c62d105e5266026b47d7e4dfb4ce2d03ac6cbcdbc0160aa60", + "root/usr/include/boost/asio/compose.hpp": "5fa98287529632714d7e46c8da09312ab357a2b7afb026208fd83621f6abdde7", + "root/usr/lib/aarch64-linux-gnu/libclang-cpp.so.14": "b9cdd42ec88a2655f2ff005738dfcfe4cab08a188daa4ebabb200251d6773d84", + "root/usr/lib/aarch64-linux-gnu/libLLVM-14.so.1": "1ff1243403be3b8dd5120198e13212f69256d4399d39271dc00bc04f24393dca" + }, + "diagnostics": [ + { + "check": "bugprone-use-after-move", + "primary": { + "path": "boost/beast/http/impl/read.hpp", + "offset": 6729, + "line": 248, + "column": 61, + "message": "'self' used after it was moved", + "replacements": [] + }, + "notes": [ + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 6693, + "line": 248, + "column": 25, + "message": "move occurred here", + "replacements": [] + }, + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 6729, + "line": 248, + "column": 61, + "message": "the use happens in a later loop iteration than the move", + "replacements": [] + } + ], + "level": "Warning" + }, + { + "check": "bugprone-use-after-move", + "primary": { + "path": "boost/beast/http/impl/read.hpp", + "offset": 6786, + "line": 251, + "column": 13, + "message": "'self' used after it was moved", + "replacements": [] + }, + "notes": [ + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 5590, + "line": 215, + "column": 24, + "message": "move occurred here", + "replacements": [] + } + ], + "level": "Warning" + }, + { + "check": "bugprone-use-after-move", + "primary": { + "path": "boost/beast/http/impl/read.hpp", + "offset": 7674, + "line": 287, + "column": 41, + "message": "'self' used after it was moved", + "replacements": [] + }, + "notes": [ + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 7654, + "line": 287, + "column": 21, + "message": "move occurred here", + "replacements": [] + }, + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 7674, + "line": 287, + "column": 41, + "message": "the use happens in a later loop iteration than the move", + "replacements": [] + } + ], + "level": "Warning" + }, + { + "check": "bugprone-use-after-move", + "primary": { + "path": "boost/beast/http/impl/read.hpp", + "offset": 8090, + "line": 301, + "column": 51, + "message": "'self' used after it was moved", + "replacements": [] + }, + "notes": [ + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 8023, + "line": 300, + "column": 25, + "message": "move occurred here", + "replacements": [] + }, + { + "path": "boost/beast/http/impl/read.hpp", + "offset": 8090, + "line": 301, + "column": 51, + "message": "the use happens in a later loop iteration than the move", + "replacements": [] + } + ], + "level": "Warning" + } + ], + "proof": { + "original_source": "6a1c02674c45bc4b9fcbf6d17a72226fc074e9a4", + "failed_ref": "fae1741f-4e00-446f-a5eb-3540ddcf1016", + "reduction_ref": "b40d5db8-1a12-4d60-8d04-3d757ccbf877", + "reduction_sha256": "f5aebc7aed2ff847ee92415484c3e98581a8afb5f7cf6273dc93fa486f9a633f", + "reason": "Asio stackless YIELD transfers handler and exits invocation; reentry binds moved-to handler. Clang14 flow-sensitive use-after-move does not correlate this macro control path.", + "source_sha256": { + "cpp/src/session.cpp": "5f6ece8ae6ddefb9363a6f6ee846bd6f75d7f30b090917a3f668e5604cc17590", + "cpp/src/server.cpp": "bf83d9b3fdfb67b0015f64f602fecacd1f58a66da42bea2d484467ad71f91531" + } + }, + "policy_sha256": "6711fc1ae69b8fea751850754030df2c882b7ed8610d5e43bea173515a331966" +} diff --git a/cpp/docs/static-analysis.md b/cpp/docs/static-analysis.md new file mode 100644 index 000000000..27e0b0b73 --- /dev/null +++ b/cpp/docs/static-analysis.md @@ -0,0 +1,80 @@ +# Static analysis + +The pilot uses the qualified SDK's clang-tidy 14.0.6 and Clang Static Analyzer on the actual GCC 12.2 Pi C++20 compilation model. The existing core and C++ check families, `bugprone-dangling-handle` and `bugprone-use-after-move` remain selected. The two added checks are `clang-analyzer-unix.Malloc` and `clang-analyzer-unix.MismatchedDeallocator`. New or actionable findings are fatal; only the four exact pinned reports documented below can be classified, with their raw failed child exit preserved. `HeaderFilterRegex: ".*"` and `--system-headers` preserve emitted header diagnostics and caller paths; there is no vendor-directory exclusion or automatic fix. + +This selection follows the requested GPT Pro recommendation from 2026-11-13, checked against the installed engine and representative defect controls. It adds modeled C allocation-family leak, use-after-free and deallocation mismatch coverage to the already enabled C++ new/delete checks. Cppcheck, GCC `-fanalyzer`, alpha checks, ownership annotation rollout and SDK upgrades are deferred. + +## Commands + +Use the same read-only qualified SDK as the ordinary C++ build; no install, download or separate dependency graph is needed. + +```sh +python3 cpp/tools/build.py /absolute/qualified-sdk pi-release configure +python3 cpp/tools/build.py /absolute/qualified-sdk pi-release tidy +python3 cpp/tools/build.py /absolute/qualified-sdk pi-release complete +``` + +The `complete` Pi workflow enforces analysis after its build and six CTests. `tidy` delegates to `cpp/tools/analyze.py`; the always-run CMake target `cpp-static-analysis` invokes that same runner. An unchanged Ninja tree does not make the explicit analysis target up to date. The native sanitizer workflow remains separate. + +For focused validation after configuration: + +```sh +python3 cpp/tools/analyze.py /absolute/qualified-sdk pi-release controls +python3 cpp/tools/analyze.py /absolute/qualified-sdk pi-release check +python3 cpp/tests/analysis/test_analyze.py /absolute/qualified-sdk +``` + +The default runner mode includes controls and the complete source scan. `controls` runs the three static fixtures; `check` scans the thirteen production/test TUs after metadata and coverage validation. The wrapper tests additionally exercise missing tools/config/database, an unsupported checker, substituted or duplicate compilation entries, a malformed frontend input, invalid profiles, timeout and process failure. Deliberately incorrect C++ fixtures are never linked or executed. + +## Coverage and receipts + +The Pi preset enables an `OBJECT EXCLUDE_FROM_ALL` fixture target solely to generate accurate compilation commands. Its three TUs are separate from the twelve production TUs and `protocol_tests.cpp`: sixteen unique database entries, thirteen real roots and three controls. Source membership, compiler/sysroot, C++20 mode and lane paths must match; a file count alone is insufficient. Missing or foreign entries fail before analysis. Reconfigure after relocating a lane; do not rewrite an old database. + +Every configured positive check pattern must match the installed full checker catalog, and all required IDs must appear in the effective selected listing. Clang 14 can return zero from a listing that silently omits an unknown name, so process success alone is insufficient. + +Each TU gets a serial child, raw stdout/stderr, status, fixes YAML when emitted, and one fresh `--enable-check-profile` / `--store-check-profile` JSON naming that exact source. A profile proves the AST consumer was invoked; it does not prove parsing succeeded or every path was explored. Frontend errors, fallback/skip messages, crashes, timeouts and malformed or missing profiles remain failures. Pinned Clang 14 legitimately omits fixes YAML when no selected diagnostic exists; the wrapper records that absence and still requires the profile and frontend/status checks. It never invents a clean YAML report. + +Receipts under `.data/cpp-evidence/analysis-*/` bind the Git revision/status, configuration, compilation database, tool, source hashes, expected and processed roots, commands, raw paths and elapsed time. The ordinary build driver retains its additional disk/timing receipt. Child RSS is labeled as the cumulative `RUSAGE_CHILDREN` maximum and is distinct from an independently measured cgroup peak. + +## Controls and resource limits + +The dirty fixture requires exactly seven diagnostic IDs at independently marked locations: a malloc leak, a new leak, use after delete, mismatched release in a synthetic system header with a first-party allocation path, an uninitialized ordinary-call argument, and a genuine same-invocation smart-pointer move reported by both `bugprone-use-after-move` and `clang-analyzer-cplusplus.Move`. Clean ownership and named-coroutine cancellation-pattern TUs each require a separate zero exit and fresh source-matched profile. The cancellation fixture is static input only; it is not a new executed cancellation proof. The six existing native ASan/UBSan/leak CTests retain the actual socket, cancellation, control, restart and resource evidence. + +The final v4 profile requires eleven wrapper failure/mutation tests and three separately counted real Clang control TUs, including exactly seven expected dirty-control diagnostic records at the marked locations. The ownership and named-coroutine controls remain clean and genuine first-party same-invocation move misuse stays fatal. Earlier writer-only measurements are development evidence and are not relabelled as final canonical acceptance. Final measured qualification is recorded separately below. + +Jobs remain serial. Each analyzer child uses the existing 1 GiB virtual-address limiter, with an approximately 300-second deadline for a complete control or source phase. The build driver preserves the 256 MiB incremental disk reserve, 768 MiB additional-output bound, 2,000,000,000-byte SDK/candidate ceiling and 2 GiB free-space floor. The external acceptance capture separately verifies physical cgroup limits, peak and OOM/swap counters. Do not apply the compiler address-space ceiling to an ASan runtime's shadow mappings. + +## Limits and ownership + +The first canonical scan processed all thirteen real TUs and failed on four `bugprone-use-after-move` reports in pinned Beast `http/impl/read.hpp` at 248:61, 251:13, 287:41 and 301:51. Its raw exit1 receipt remains failed. Source inspection and one reduced actual-Asio static probe established that `BOOST_ASIO_CORO_YIELD` transfers the handler and exits that invocation before later use; callback reentry binds the moved-to handler. Clang14's flow-sensitive, non-path-sensitive move checker does not correlate that macro resume path. The equivalent explicit-return operation was clean, while genuine same-invocation misuse remained fatal under both move checks. + +`clang14-beast-read-baseline.json` classifies only those four exact reports from the Session TU: checker, primary SDK-relative path/offset/line/column/message, every move/loop note, warning level and cardinality must match. The loader, clang-tidy, Clang/LLVM frontend libraries, three related SDK headers and check-policy hashes must also match. Session/Server hashes are provenance, not eligibility constraints. The runner preserves all raw exit1/status/YAML/stdout/stderr/profile evidence and reports `classified:4`, `new:0` separately; it does not claim zero warnings. New or changed findings, first-party primaries, missing/stale entries, identity drift and unclassified failed exits remain fatal. Controls never receive this classification. Mutation tests and the actual genuine-move positive keep that boundary explicit. The original failure is `fae1741f-4e00-446f-a5eb-3540ddcf1016`; reduced proof is `b40d5db8-1a12-4d60-8d04-3d757ccbf877`. The corrected canonical scan and native acceptance are qualified only by the final exact-source measurements below; the original failed scan remains failed. + +These heap models do not establish POSIX descriptor, pipe/socket or arbitrary OpenSSL-resource lifetime safety. They do not prove absence of shared-pointer cycles, retained reachable owners, post-cancellation use-after-free or every exceptional teardown. Clang 14 creates sink/aborted paths for coroutine nodes; a clean coroutine control and profile cannot overcome that engine limitation. Preserve explicit owner/canceller/join reasoning and dynamic evidence. + +`BudgetResource` pairs aligned `operator new` and `operator delete`; JSON arenas outlive their values. State file descriptors use the existing RAII owner or explicit close-on-exception paths. Sessions retain complete write buffers across awaited writes and join downstream close observation before releasing storage. Transport awaits retain shared implementation owners, while deadline callbacks use weak ownership. Control frame buffers are initialized and owned across suspension, and completion retains Control. No runtime ownership code is changed by this tooling unit. + +The earlier Boost/Control `core.CallAndMessage` and `core.NullDereference` reports disappeared after combined owner/initialization changes. Their cause was not isolated. That history is neither a proven SDK false positive nor permission to suppress SDK-primary findings; retain their complete first-party allocation/call paths for diagnosis. + +## Measured qualification + +Final qualification binds source `959bc0c643b0a1611313087665a964988358e37e`, tree `9d7eade3059069f242eec7bd9c7816995c01361b`. Runtime `cpp/src/**` and `cpp/include/**` are unchanged from the shipped pilot; this is analysis/configuration/control-fixture enforcement. Later documentation commits retain the analyzed-source identity and must prove unchanged runtime/tooling inputs. + +The first canonical scan at `6a1c02674c45bc4b9fcbf6d17a72226fc074e9a4` correctly FAILED after all thirteen real TUs: capture233.814 seconds, cgroup peak568,934,400B. Its four Beast diagnostics and raw exit1 remain historical negative evidence under `fae1741f-4e00-446f-a5eb-3540ddcf1016`; this is not a successful v4 result. The reduced Asio probe `b40d5db8-1a12-4d60-8d04-3d757ccbf877` is a separate diagnostic experiment. + +| Final v4 workload | Workload wall s | Child max RSS KiB | Cgroup peak B | Allocated-output delta B | +| ------------------------------------------------ | --------------- | ----------------- | ------------- | ------------------------ | +| Eleven wrapper tests + three real Clang controls | 20.199 | 326500 | 291459072 | 491520 | +| Complete thirteen-real-TU scan | 219.434 | 599148 | 565817344 | 339968 | +| Native sanitizer configure/build checkpoint | 143.558 | 921620 | 1068220416 | 133648384 | +| Six native sanitizer CTests | 26.093 | 124876 | 216006656 | 487424 | + +All phases exited0 with actual serial1GiB cgroup/swap0/CPU1 limits; memory.max/OOM/oom_kill and swap counters remained0. The native build peak1,068,220,416B was near its1GiB cap, not a measurement of uncapped demand. Child `RUSAGE_CHILDREN` RSS and aggregate cgroup/file-cache charge are different and not additive. Allocated-output deltas are before/after snapshots, not transient disk peaks; final free2,673,020,928B remained above2GiB. Existing SDK/dependencies were reused, with no install or extra graph. ASan runtime shadow mappings were not given the analyzer's virtual-address ceiling. + +The table uses the workload/controller resource clock consistently. Evidence capture includes setup/teardown: controls20.538s, scan219.879s, native checkpoint143.953s and CTest26.789s. The raw CTest report was23.61s and passed exactly6/6: protocol1.23s, socket6.84s, disconnect5.80s, control5.86s, recovery1.87s and resources2.01s. Those different clocks are retained, not mixed. Native flags include `-fsanitize=address,undefined -fno-sanitize-recover=undefined -fno-omit-frame-pointer`, with leak detection, halt/abort and UBSan halt/stacktrace enabled through the existing candidate environment. + +The accepted source scan processed all13 roots with fresh source-matched frontend profiles: twelve raw child exits0, Session raw child exit1 with exactly four retained diagnostics. Final policy status was `classified:4`, `new:0`, `missing:0`, `unclassified_exit:0`; this is **not zero warnings**. The three controls detected all seven expected dirty diagnostic records and kept both ownership/coroutine controls clean; genuine first-party same-invocation misuse remained fatal. Eleven wrapper tests include malformed report, baseline fingerprint/count/path/check/hash mutations and process/input failures. Emitted fixes YAML was fully parsed with the pinned Bun built-in YAML parser; absent YAML is legitimate only with otherwise qualified clean processing. + +Every registered phase ran `/usr/bin/python3 /tmp/ai-cpp-static-analysis-20261113/final-verification-v4/run.py ` from the exact canonical worktree; that capture driver invokes the committed command modes above and binds its argv/input hashes. Repository evidence namespace is `c18ba89e18e8c4c810e8dfab831978ea7ed5aa33d6f343bf3282dea4e12db13c`: controls `1619f071-0ef6-45a6-8161-277b224c8b32`, scan `e03a2c80-b652-47dc-809a-b2a117f8d0c6`, native checkpoint `9bb1ae87-45fa-4a6d-b109-94f261f12af4`, native CTest `e4f2264e-f9b7-4712-8d15-d86de714436d`. Cheap/configure receipts and all six actual cgroup teardown proofs are retained in the same packet; units are not-found/inactive/dead and cgroups absent. + +Source/tooling input manifest SHA256 is `d7d114fdbbc89b601bd6901784bc0085250602ffbef13af2a3beba648e06a2c0`; strict scan qualification SHA256 `8594f0a45cbd00d551e1cb572a6dac4b3c22014fa98255276b7ee4aeaa646032`; phase-resource summary SHA256 `f3987ced7dc1996a8f92e68c84f1d73c253f5f1df1139931933e0636158c991d`; cleanup SHA256 `c513dc7c885726d96326bce355e79cd2f966e677575d96d961d4bbe896a49b14`. The checked-in baseline also binds exact tool/frontend/SDK-header/check-policy hashes; no broad suppression, fallback compilation or source/runtime rewrite is authorized. Static analysis and clean profiles do not prove shared-cycle, POSIX-FD or deferred coroutine/cancellation lifetime safety; retain dynamic sanitizer/socket evidence and explicit ownership reasoning. diff --git a/cpp/tests/analysis/clean_cancel.cpp b/cpp/tests/analysis/clean_cancel.cpp new file mode 100644 index 000000000..83ab103ea --- /dev/null +++ b/cpp/tests/analysis/clean_cancel.cpp @@ -0,0 +1,60 @@ +// Named coroutine/real-pipe ownership adapted from the qualified isolated probe. +// Static input only; existing sanitizer socket tests exercise actual cancellation. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +namespace asio=boost::asio; +struct Fd { + int value; + explicit Fd(int fd):value(fd){} + ~Fd(){if(value>=0)::close(value);} + Fd(Fd const&)=delete; + Fd& operator=(Fd const&)=delete; +}; +struct Control { + asio::posix::stream_descriptor input; + bool entered=false,completed=false; + unsigned completions=0; + explicit Control(asio::io_context& io):input(io){} +}; +asio::awaitable read_frame(std::shared_ptr owner) { + std::array prefix{}; + owner->entered=true; + boost::system::error_code error; + co_await asio::async_read(owner->input,asio::buffer(prefix),asio::redirect_error(asio::use_awaitable,error)); + if(error!=asio::error::operation_aborted)throw std::runtime_error("Expected cancellation"); +} +asio::awaitable named_session(std::shared_ptr owner){co_await read_frame(owner);} +} + +void clean_pending_read_cancellation() { + asio::io_context io; + int pipes[2]={-1,-1}; + if(::pipe(pipes)!=0)throw std::runtime_error("pipe failed"); + Fd read_fd(pipes[0]),write_fd(pipes[1]); + auto owner=std::make_shared(io); + owner->input.assign(read_fd.value); + read_fd.value=-1; + asio::co_spawn(io,named_session(owner),[owner](std::exception_ptr error){ + if(error)std::rethrow_exception(error); + owner->completed=true;++owner->completions; + }); + asio::post(io,[owner]{ + if(!owner->entered)throw std::runtime_error("Cancellation before pending read"); + owner->input.cancel(); + }); + io.run(); + if(!owner->completed||owner->completions!=1)throw std::runtime_error("Completion must drain once"); +} diff --git a/cpp/tests/analysis/clean_ownership.cpp b/cpp/tests/analysis/clean_ownership.cpp new file mode 100644 index 000000000..1df195f39 --- /dev/null +++ b/cpp/tests/analysis/clean_ownership.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include +#include + +namespace { +struct Descriptor { + int value; + explicit Descriptor(int fd):value(fd){} + ~Descriptor(){if(value>=0)::close(value);} + Descriptor(Descriptor const&)=delete; + Descriptor& operator=(Descriptor const&)=delete; +}; +} + +int clean_unique_ptr(bool refuse) { + auto pointer = std::make_unique(42); + if (refuse) return -1; + return *pointer; +} + +bool clean_descriptor(bool refuse) { + Descriptor fd(::open("/dev/null",O_RDONLY|O_CLOEXEC)); + if (refuse) return false; + return fd.value>=0; +} + +int clean_shared_callback() { + auto owner=std::make_shared(42); + std::weak_ptr observer=owner; + std::vector> callbacks; + callbacks.emplace_back([owner]{return *owner;}); + owner.reset(); + int result=callbacks.front()(); + callbacks.clear(); + return observer.expired()?result:0; +} diff --git a/cpp/tests/analysis/fixture_sdk/release.hpp b/cpp/tests/analysis/fixture_sdk/release.hpp new file mode 100644 index 000000000..d6b7f2715 --- /dev/null +++ b/cpp/tests/analysis/fixture_sdk/release.hpp @@ -0,0 +1,5 @@ +#pragma once +#include +inline void fixture_sdk_release(int* pointer) { + std::free(pointer); // expect-mismatch +} diff --git a/cpp/tests/analysis/raw_defects.cpp b/cpp/tests/analysis/raw_defects.cpp new file mode 100644 index 000000000..0b7e6ede3 --- /dev/null +++ b/cpp/tests/analysis/raw_defects.cpp @@ -0,0 +1,43 @@ +// Analysis input only: never compile, link or execute these deliberate defects. +#include +#include +#include +#include + +int malloc_branch(bool cleanup) { + int* pointer = static_cast(std::malloc(sizeof(int))); + if (!pointer) return 0; + if (cleanup) { std::free(pointer); return 1; } + return 0; // expect-malloc +} + +int new_branch(bool abandon) { + int* pointer = new int(42); + if (abandon) return 0; // expect-new-leak + int result = *pointer; + delete pointer; + return result; +} + +int stale_new_pointer() { + int* pointer = new int(42); + delete pointer; + return *pointer; // expect-new-uaf +} + +void mismatched_release() { + int* pointer = new int(42); // allocation-owned + fixture_sdk_release(pointer); +} + +void consume_value(int); +void uninitialized_argument() { + int value; + int* alias = &value; + consume_value(*alias); // expect-uninitialized +} + +void genuine_same_dispatch(std::unique_ptr self) { + auto destination=std::move(self); + *self=7; // expect-genuine-move +} diff --git a/cpp/tests/analysis/test_analyze.py b/cpp/tests/analysis/test_analyze.py new file mode 100644 index 000000000..92502f9cd --- /dev/null +++ b/cpp/tests/analysis/test_analyze.py @@ -0,0 +1,228 @@ +#!/usr/bin/python3 +"""Real engine controls and fail-closed wrapper checks; never execute C++ input.""" +import importlib.util +import copy +import json +from pathlib import Path +import signal +import sys +import tempfile +import unittest +from unittest.mock import patch + +WORKSPACE = Path(__file__).resolve().parents[3] +SDK = Path(sys.argv[1]).resolve() +SPEC = importlib.util.spec_from_file_location("pilot_analysis", WORKSPACE / "cpp/tools/analyze.py") +ANALYSIS = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(ANALYSIS) + + +def write_policy(path, value): + path.write_text("\n".join(key + ": " + json.dumps(scalar) for key, scalar in value.items()) + "\n") + + +class AnalysisChecks(unittest.TestCase): + @classmethod + def setUpClass(cls): + temporary = WORKSPACE / ".data/cpp-tmp/analysis-tests" + temporary.mkdir(parents=True, exist_ok=True) + cls.output = Path(tempfile.mkdtemp(prefix="controls-", dir=temporary)) + cls.config = WORKSPACE / "cpp/.clang-tidy" + cls.build = WORKSPACE / ".data/cpp-build/pi-release" + cls.database = cls.build / "compile_commands.json" + cls.configuration = ANALYSIS.policy(cls.config) + metadata = cls.output / "metadata" + metadata.mkdir() + cls.tool = ANALYSIS.Tool(SDK, WORKSPACE, metadata) + cls.metadata = cls.tool.metadata(cls.configuration, cls.config) + cls.catalog = ANALYSIS.check_names((metadata / "catalog.stdout").read_text()) + cls.baseline_path = WORKSPACE / "cpp/clang14-beast-read-baseline.json" + cls.baseline = ANALYSIS.beast_baseline(cls.baseline_path, SDK, WORKSPACE) + + def control(self, name, source): + output = self.output / name + output.mkdir() + source = Path(source) + command = [str(SDK / "root/usr/bin/aarch64-linux-gnu-g++-12"), "--sysroot=" + str(SDK / "root"), + "-B" + str(SDK / "host-tools/bin") + "/", "-std=c++20", "-O2", "-g0", "-c", str(source), + "-o", str(output / "never-produced.o")] + (output / "compile_commands.json").write_text(json.dumps([{"directory": str(output), "file": str(source), "arguments": command}])) + result = ANALYSIS.Tool(SDK, WORKSPACE, output).analyze(source, output, self.config, name, 20) + self.assertFalse((output / "never-produced.o").exists(), "Static controls must never compile or execute a program") + return result + + def test_complete_actual_compilation_model(self): + units = ANALYSIS.compilation_units(self.database, WORKSPACE, self.build, SDK) + self.assertEqual(len(units), 16) + self.assertIn(WORKSPACE / "cpp/tests/protocol_tests.cpp", units) + self.assertEqual(sum(source.parent == WORKSPACE / "cpp/tests/analysis" for source in units), 3) + + def test_missing_artifacts(self): + with self.assertRaises(FileNotFoundError): + ANALYSIS.Tool(self.output / "missing-sdk", WORKSPACE, self.output) + with self.assertRaises(FileNotFoundError): + ANALYSIS.policy(self.output / "missing-policy") + with self.assertRaises(FileNotFoundError): + ANALYSIS.compilation_units(self.output / "missing-db", WORKSPACE, self.build, SDK) + + def test_invalid_policy_and_unknown_checker(self): + path = self.output / "invalid-policy.json" + value = dict(self.configuration, WarningsAsErrors="") + write_policy(path, value) + with self.assertRaises(ValueError): + ANALYSIS.policy(path) + unknown = dict(self.configuration, Checks="-*,clang-analyzer-cplusplus.NewDeleteLeaks,uos-not-an-installed-check") + write_policy(path, unknown) + output = self.output / "unknown-check" + output.mkdir() + tool = ANALYSIS.Tool(SDK, WORKSPACE, output) + with self.assertRaisesRegex(ValueError, "Unsupported required"): + tool.metadata(ANALYSIS.policy(path), path) + self.assertEqual(json.loads((output / "catalog.json").read_text())["exit"], 0, + "A metadata command's success must not hide a misspelled required checker") + + def test_compilation_coverage_failures(self): + original = json.loads(self.database.read_text()) + cases = {"missing": original[:-1], "duplicate": original[:-1] + [original[0]], + "foreign-directory": [dict(original[0], directory=str(self.output))] + original[1:], + "foreign-source": [dict(original[0], file=str(self.output / "foreign.cpp"))] + original[1:], + "wrong-standard": [dict(original[0], command=original[0]["command"].replace("-std=c++20", "-std=c++17"))] + original[1:]} + for name, entries in cases.items(): + with self.subTest(name=name): + path = self.output / (name + ".json") + path.write_text(json.dumps(entries)) + with self.assertRaises(ValueError): + ANALYSIS.compilation_units(path, WORKSPACE, self.build, SDK) + + def test_three_real_controls_and_exact_findings(self): + units = ANALYSIS.compilation_units(self.database, WORKSPACE, self.build, SDK) + fixtures = [source for source in units if source.parent == WORKSPACE / "cpp/tests/analysis"] + results = ANALYSIS.controls(self.tool, fixtures, self.build, self.config) + self.assertEqual(len(results), 3) + for result in results: + self.assertEqual(Path(result["consumer_profile"]["file"]).resolve(), Path(result["source"]).resolve()) + if Path(result["source"]).name != "raw_defects.cpp": + self.assertEqual(result["exit"], 0) + self.assertEqual(result["diagnostic_ids"], []) + self.assertIsNone(result["report"]) + self.assertEqual(result["report_status"], "not-emitted-for-clean-input") + else: + self.assertIn("bugprone-use-after-move", result["diagnostic_ids"]) + self.assertIn("clang-analyzer-cplusplus.Move", result["diagnostic_ids"]) + self.assertEqual(ANALYSIS.classify_beast([result], self.baseline, SDK, WORKSPACE)["classified_count"], 0) + + def test_beast_baseline_rejects_identity_drift(self): + for name, key in [("tool", "root/usr/bin/clang-tidy-14"), ("frontend", "root/usr/lib/aarch64-linux-gnu/libclang-cpp.so.14"), + ("header", "root/usr/include/boost/beast/http/impl/read.hpp"), ("policy", None)]: + with self.subTest(name=name): + value = copy.deepcopy(self.baseline) + if key is None: + value["policy_sha256"] = "0" * 64 + else: + value["sdk_sha256"][key] = "0" * 64 + path = self.output / (name + "-baseline.json") + path.write_text(json.dumps(value)) + with self.assertRaises(ValueError): + ANALYSIS.beast_baseline(path, SDK, WORKSPACE) + + def test_beast_classification_is_exact_and_finite(self): + def location(value): + return {"Message": value["message"], "FilePath": str(SDK / "root/usr/include" / value["path"]), + "FileOffset": value["offset"], "Replacements": value["replacements"]} + diagnostics = [{"DiagnosticName": value["check"], "DiagnosticMessage": location(value["primary"]), + "Notes": [location(note) for note in value["notes"]], "Level": value["level"]} + for value in self.baseline["diagnostics"]] + result = {"source": str(WORKSPACE / "cpp/src/session.cpp"), "exit": 1, "diagnostics": diagnostics} + accepted = ANALYSIS.classify_beast([result], self.baseline, SDK, WORKSPACE) + self.assertEqual((accepted["classified_count"], accepted["new_count"], accepted["missing_count"]), (4, 0, 0)) + for mutation in ("check", "offset", "message", "note", "path", "first-party", "extra", "missing", "empty-failed-exit"): + with self.subTest(mutation=mutation): + changed = copy.deepcopy(result) + first = changed["diagnostics"][0] + if mutation == "check": first["DiagnosticName"] = "bugprone-dangling-handle" + elif mutation == "offset": first["DiagnosticMessage"]["FileOffset"] += 1 + elif mutation == "message": first["DiagnosticMessage"]["Message"] += " changed" + elif mutation == "note": first["Notes"][0]["Message"] += " changed" + elif mutation == "path": first["DiagnosticMessage"]["FilePath"] = str(SDK / "root/usr/include/boost/asio/coroutine.hpp") + elif mutation == "first-party": first["DiagnosticMessage"]["FilePath"] = str(WORKSPACE / "cpp/src/session.cpp") + elif mutation == "extra": changed["diagnostics"].append(copy.deepcopy(first)) + elif mutation == "missing": changed["diagnostics"].pop() + else: changed["diagnostics"] = [] + classified = ANALYSIS.classify_beast([changed], self.baseline, SDK, WORKSPACE) + self.assertTrue(classified["new_count"] or classified["missing_count"] or classified["unclassified_exit_count"]) + + def test_missing_malformed_and_wrong_source_profiles(self): + directory = self.output / "invalid-profiles" + directory.mkdir() + source = WORKSPACE / "cpp/tests/protocol_tests.cpp" + with self.assertRaises(ValueError): + ANALYSIS.consumer_profile(directory, source) + path = directory / "bad.json" + path.write_text("not-json") + with self.assertRaises(ValueError): + ANALYSIS.consumer_profile(directory, source) + path.write_text(json.dumps({"file": str(self.output / "foreign.cpp"), "timestamp": "test", "profile": {}})) + with self.assertRaises(ValueError): + ANALYSIS.consumer_profile(directory, source) + + def test_frontend_failure_is_not_a_finding_pass(self): + source = self.output / "invalid.cpp" + source.write_text("int invalid = ;\n") + result = self.control("parse-error", source) + self.assertNotEqual(result["exit"], 0) + self.assertFalse(result["frontend_success"]) + self.assertIn("clang-diagnostic-error", result["diagnostic_ids"]) + + def test_timeout_and_crash_preserve_failure(self): + with self.assertRaises(TimeoutError): + ANALYSIS.execute([sys.executable, "-c", "import time; time.sleep(5)"], WORKSPACE, self.output, "timeout", 0.05) + timeout = json.loads((self.output / "timeout.json").read_text()) + self.assertTrue(timeout["timed_out"]) + self.assertNotEqual(timeout["exit"], 0) + result = ANALYSIS.execute([sys.executable, "-c", "import os,signal; os.kill(os.getpid(),signal.SIGTERM)"], + WORKSPACE, self.output, "crash", 5) + self.assertEqual(result["exit"], -signal.SIGTERM) + + def test_emitted_report_must_be_complete_and_match_raw_diagnostics(self): + source = WORKSPACE / "cpp/tests/protocol_tests.cpp" + header = "---\nMainSourceFile: '" + str(source) + "'\n" + cases = [("truncated", header + "Diagnostics: [\n", "", True), + ("unterminated", header + "Diagnostics: []\n", "", True), + ("wrong-type", header + "Diagnostics: {}\n...\n", "", True), + ("raw-disagrees", header + "Diagnostics: []\n...\n", + str(source) + ":1:1: error: defect [clang-analyzer-unix.Malloc,-warnings-as-errors]\n", True), + ("explicit-empty", header + "Diagnostics: []\n...\n", "", False)] + original_execute = ANALYSIS.execute + for name, document, raw, must_fail in cases: + with self.subTest(name=name): + output = self.output / ("report-" + name) + output.mkdir() + def fake_tool(argv, cwd, directory, label, timeout): + if not any(str(value).startswith("--export-fixes=") for value in argv): + return original_execute(argv, cwd, directory, label, timeout) + report = Path(next(value.split("=", 1)[1] for value in argv if value.startswith("--export-fixes="))) + report.write_text(document) + profiles = Path(next(value.split("=", 1)[1] for value in argv if value.startswith("--store-check-profile="))) + (profiles / "consumer.json").write_text(json.dumps({"file": str(source), "timestamp": "synthetic", "profile": {}})) + stdout, stderr = directory / (label + ".stdout"), directory / (label + ".stderr") + stdout.write_text(raw) + stderr.write_text("") + return {"exit": 0, "stdout": str(stdout), "stderr": str(stderr)} + tool = ANALYSIS.Tool(SDK, WORKSPACE, output) + with patch.object(ANALYSIS, "execute", side_effect=fake_tool): + if must_fail: + with self.assertRaises(ValueError): + tool.analyze(source, self.build, self.config, name, 5) + else: + result = tool.analyze(source, self.build, self.config, name, 5) + self.assertEqual(result["diagnostic_ids"], []) + self.assertEqual(result["report_status"], "exported") + self.assertIsNotNone(result["report"]) + + @classmethod + def tearDownClass(cls): + print(json.dumps({"controls": str(cls.output), "metadata": cls.metadata, "cpp_programs_executed": False})) + + +if __name__ == "__main__": + unittest.main(argv=[sys.argv[0], *sys.argv[2:]], verbosity=2) diff --git a/cpp/tools/analyze.py b/cpp/tools/analyze.py new file mode 100644 index 000000000..60a53a060 --- /dev/null +++ b/cpp/tools/analyze.py @@ -0,0 +1,396 @@ +#!/usr/bin/python3 +"""Bounded offline clang-tidy gate for the pilot's actual compilation model.""" +import fnmatch +from collections import Counter +import hashlib +import json +import os +from pathlib import Path +import re +import resource +import shlex +import signal +import subprocess +import sys +import time + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def policy(path): + # This owned policy is deliberately limited to flat JSON scalar values in YAML. + value = {} + for line in Path(path).read_text().splitlines(): + match = re.fullmatch(r"([A-Za-z]+): (.+)", line) + if not match or match[1] in value: + raise ValueError("Analysis policy requires unique flat scalar settings") + value[match[1]] = json.loads(match[2]) + if set(value) != {"Checks", "WarningsAsErrors", "InheritParentConfig", "HeaderFilterRegex"}: + raise ValueError("Analysis policy must contain only explicit check/failure settings") + if value["WarningsAsErrors"] != "*" or value["InheritParentConfig"] is not False or value["HeaderFilterRegex"] != ".*": + raise ValueError("Analysis must retain fatal diagnostics and isolated configuration") + patterns = value["Checks"].split(",") + if patterns[0] != "-*" or len(patterns) < 2 or any(not p or p.startswith("-") for p in patterns[1:]): + raise ValueError("Analysis policy requires an explicit positive check list after -*") + return value + + +def check_names(output): + lines = output.splitlines() + if not lines or lines[0] != "Enabled checks:": + raise ValueError("Malformed clang-tidy checker inventory") + names = [line.strip() for line in lines[1:] if line.strip()] + if not names or len(set(names)) != len(names) or any(not re.fullmatch(r"[a-zA-Z0-9_.-]+", name) for name in names): + raise ValueError("Empty, duplicate or malformed clang-tidy checker inventory") + return set(names) + + +def supported_checks(configuration, available): + selected = set() + for pattern in configuration["Checks"].split(",")[1:]: + matches = {name for name in available if fnmatch.fnmatchcase(name, pattern)} + if not matches: + raise ValueError("Unsupported required clang-tidy check: " + pattern) + selected.update(matches) + return selected + + +def compilation_units(path, workspace, build, sdk): + entries = json.loads(Path(path).read_text()) + expected = set((workspace / "cpp/src").glob("*.cpp")) | {workspace / "cpp/tests/protocol_tests.cpp"} + expected |= {workspace / "cpp/tests/analysis" / name for name in ("raw_defects.cpp", "clean_ownership.cpp", "clean_cancel.cpp")} + if not isinstance(entries, list) or len(entries) != len(expected): + raise ValueError("Compilation database must cover every production and protocol-test TU exactly once") + found = set() + for entry in entries: + source = Path(entry["file"]) + if source not in expected or source in found or not source.is_file() or Path(entry["directory"]) != build: + raise ValueError("Duplicate, unexpected, missing or foreign compilation unit") + if ("arguments" in entry) == ("command" in entry): + raise ValueError("Each compilation entry needs exactly one command representation") + argv = entry["arguments"] if "arguments" in entry else shlex.split(entry["command"]) + if not isinstance(argv, list) or not argv or not all(isinstance(value, str) for value in argv): + raise ValueError("Invalid compilation command") + compiler = sdk / "root/usr/bin/aarch64-linux-gnu-g++-12" + if Path(argv[0]) != compiler or "--sysroot=" + str(sdk / "root") not in argv or "-std=c++20" not in argv: + raise ValueError("Analysis requires the pinned Pi C++20 compiler/sysroot model") + if "-c" not in argv or argv[argv.index("-c") + 1] != str(source): + raise ValueError("Compilation command does not name its source") + found.add(source) + if found != expected: + raise ValueError("Compilation database coverage differs from the first-party source set") + return sorted(found) + + +def consumer_profile(directory, source): + profiles = list(directory.glob("*.json")) + if len(profiles) != 1: + raise ValueError("Each TU requires exactly one fresh AST-consumer profile") + value = json.loads(profiles[0].read_text()) + if not isinstance(value, dict) or Path(value["file"]).resolve() != source.resolve(): + raise ValueError("AST-consumer profile names a different source") + if not isinstance(value.get("profile"), dict) or not isinstance(value.get("timestamp"), str): + raise ValueError("Malformed AST-consumer profile") + return {"path": str(profiles[0]), "sha256": digest(profiles[0]), "file": value["file"], "timestamp": value["timestamp"]} + + +def diagnostic_report(path, source, raw, workspace, output, label, timeout): + exported = path.read_text() + if not exported.startswith("---\n") or not exported.rstrip().endswith("\n..."): + raise ValueError("Incomplete clang-tidy YAML document") + common = Path(subprocess.check_output(["git", "rev-parse", "--path-format=absolute", "--git-common-dir"], + cwd=workspace, text=True).strip()).parent + bun = common / ".data/runtimes/bun/linux-arm64/1.4.0-086c4121c8738a8e0f5ed730e8a461bc3973b4444e372ddb77aef9a747fa2ae9/bun" + if not bun.is_file(): + raise FileNotFoundError("The pinned Bun YAML parser is unavailable") + script = ('if(Bun.version!=="1.4.0"||typeof Bun.YAML?.parse!=="function")throw new Error("Pinned YAML parser unavailable");' + 'console.log(JSON.stringify(Bun.YAML.parse(await Bun.file(process.argv[1]).text())));') + parsed = execute([str(bun), "--no-env-file", "--no-install", "--eval", script, str(path)], + workspace, output, label + "-yaml", min(timeout, 15)) + if parsed["exit"] != 0 or Path(parsed["stderr"]).read_bytes(): + raise ValueError("Malformed clang-tidy YAML report; parser output retained") + value = json.loads(Path(parsed["stdout"]).read_text()) + if not isinstance(value, dict) or set(value) != {"MainSourceFile", "Diagnostics"}: + raise ValueError("Invalid clang-tidy report structure") + if value["MainSourceFile"] != str(source) or not isinstance(value["Diagnostics"], list): + raise ValueError("Source-mismatched report or invalid Diagnostics payload") + names = [] + for entry in value["Diagnostics"]: + if not isinstance(entry, dict) or not isinstance(entry.get("DiagnosticName"), str): + raise ValueError("Malformed diagnostic entry") + message = entry.get("DiagnosticMessage") + if (not isinstance(message, dict) or not isinstance(message.get("Message"), str) + or not isinstance(message.get("FilePath"), str) or type(message.get("FileOffset")) is not int + or message["FileOffset"] < 0 or not isinstance(message.get("Replacements"), list)): + raise ValueError("Malformed diagnostic message") + names.append(entry["DiagnosticName"]) + raw_names = [name for group in re.findall(r":\d+:\d+: (?:warning|error): [^\n]*\[([^\]]+)\]", raw) + for name in group.split(",") if not name.startswith("-")] + if Counter(names) != Counter(raw_names): + raise ValueError("Raw diagnostics and YAML diagnostic IDs disagree") + return names, value["Diagnostics"] + + +def beast_baseline(path, sdk, workspace): + baseline = json.loads(path.read_text()) + if baseline["schema"] != "uos-clang14-beast-read-v1" or baseline["source"] != "cpp/src/session.cpp" or len(baseline["diagnostics"]) != 4: + raise ValueError("Invalid finite Beast diagnostic baseline") + expected_sdk = {"root/lib/ld-linux-aarch64.so.1", "root/usr/bin/clang-tidy-14", + "root/usr/lib/aarch64-linux-gnu/libclang-cpp.so.14", "root/usr/lib/aarch64-linux-gnu/libLLVM-14.so.1", + "root/usr/include/boost/beast/http/impl/read.hpp", "root/usr/include/boost/asio/coroutine.hpp", + "root/usr/include/boost/asio/compose.hpp"} + if set(baseline["sdk_sha256"]) != expected_sdk: + raise ValueError("Incomplete Beast baseline identity binding") + for relative, expected in baseline["sdk_sha256"].items(): + if digest(sdk / relative) != expected: + raise ValueError("Stale Beast baseline identity: " + relative) + if digest(workspace / "cpp/.clang-tidy") != baseline["policy_sha256"]: + raise ValueError("Stale Beast baseline check policy") + return baseline + + +def beast_fingerprint(diagnostic, sdk): + if diagnostic.get("DiagnosticName") != "bugprone-use-after-move" or diagnostic.get("Level") != "Warning": + return None + def location(value): + if set(value) != {"Message", "FilePath", "FileOffset", "Replacements"} or value["Replacements"] != []: + raise ValueError("Changed Beast diagnostic location structure") + path = Path(value["FilePath"]) + relative = path.relative_to(sdk / "root/usr/include") + if str(relative) != "boost/beast/http/impl/read.hpp": + raise ValueError("Finding is outside the reviewed Beast header") + data, offset = path.read_bytes(), value["FileOffset"] + if type(offset) is not int or offset < 0 or offset >= len(data): + raise ValueError("Invalid Beast diagnostic offset") + return {"path": str(relative), "offset": offset, "line": data[:offset].count(b"\n") + 1, + "column": offset - data.rfind(b"\n", 0, offset), "message": value["Message"], "replacements": []} + try: + return {"check": diagnostic["DiagnosticName"], "primary": location(diagnostic["DiagnosticMessage"]), + "notes": [location(note) for note in diagnostic["Notes"]], "level": diagnostic["Level"]} + except (ValueError, KeyError, TypeError): + return None + + +def classify_beast(results, baseline, sdk, workspace): + expected = Counter(json.dumps(value, sort_keys=True) for value in baseline["diagnostics"]) + observed, classified, new = Counter(), [], [] + for result in results: + for index, diagnostic in enumerate(result["diagnostics"]): + fingerprint = beast_fingerprint(diagnostic, sdk) if result["source"] == str(workspace / baseline["source"]) else None + key = json.dumps(fingerprint, sort_keys=True) + item = {"source": result["source"], "diagnostic_index": index, "check": diagnostic["DiagnosticName"]} + if fingerprint is not None and observed[key] < expected[key]: + observed[key] += 1 + classified.append(item) + else: + new.append(item) + missing = sum((expected - observed).values()) + failed_exits = sum(result["exit"] != 0 and (result["exit"] != 1 or not result["diagnostics"]) for result in results) + return {"classified_count": len(classified), "new_count": len(new), "missing_count": missing, + "unclassified_exit_count": failed_exits, "classified": classified, "new": new, + "raw_child_exits": [result["exit"] for result in results]} + + +def execute(argv, cwd, output, label, timeout): + """Retain raw output, propagate timeout/crash, and settle the exact child group.""" + if timeout <= 0: + raise TimeoutError("Analysis deadline exhausted") + stdout, stderr = output / (label + ".stdout"), output / (label + ".stderr") + started = time.monotonic() + timed_out = False + with stdout.open("wb") as out, stderr.open("wb") as err: + child = subprocess.Popen(argv, cwd=cwd, env={"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "TMPDIR": str(output)}, + stdout=out, stderr=err, start_new_session=True) + try: + child.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + try: + os.killpg(child.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + child.wait(timeout=3) + except subprocess.TimeoutExpired: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait(timeout=3) + result = {"argv": list(map(str, argv)), "exit": child.returncode, "timed_out": timed_out, + "duration_s": round(time.monotonic() - started, 6), "stdout": str(stdout), "stderr": str(stderr)} + (output / (label + ".json")).write_text(json.dumps(result, indent=2) + "\n") + if timed_out: + raise TimeoutError("Analysis command timed out; raw output: " + str(stderr)) + if max(stdout.stat().st_size, stderr.stat().st_size) > 16 * 1024 * 1024: + raise ValueError("Analysis output exceeded its bound; raw files retained") + return result + + +class Tool: + def __init__(self, sdk, workspace, output): + self.sdk, self.workspace, self.output = Path(sdk), Path(workspace), Path(output) + self.binary = self.sdk / "root/usr/bin/clang-tidy-14" + loader = self.sdk / "root/lib/ld-linux-aarch64.so.1" + limit = self.workspace / "cpp/tools/compiler-limit.py" + for path in (self.binary, loader, limit): + if not path.is_file(): + raise FileNotFoundError("Required analyzer artifact is missing: " + str(path)) + libraries = str(self.sdk / "root/lib/aarch64-linux-gnu") + ":" + str(self.sdk / "root/usr/lib/aarch64-linux-gnu") + self.prefix = ["/usr/bin/python3", str(limit), str(loader), "--library-path", libraries, str(self.binary)] + + def metadata(self, configuration, config_path): + version = execute(self.prefix + ["--version"], self.workspace, self.output, "version", 15) + text = Path(version["stdout"]).read_text() + if version["exit"] != 0 or Path(version["stderr"]).read_bytes() or "LLVM version 14.0.6" not in text: + raise ValueError("The qualified clang-tidy 14.0.6 tool is unavailable") + catalog = execute(self.prefix + ["--config-file=" + str(config_path), "--checks=*", "--list-checks"], + self.workspace, self.output, "catalog", 15) + if catalog["exit"] != 0 or Path(catalog["stderr"]).read_bytes(): + raise ValueError("clang-tidy checker inventory failed") + available = check_names(Path(catalog["stdout"]).read_text()) + required = supported_checks(configuration, available) + selected = execute(self.prefix + ["--config-file=" + str(config_path), "--list-checks"], + self.workspace, self.output, "selected", 15) + if selected["exit"] != 0 or Path(selected["stderr"]).read_bytes(): + raise ValueError("clang-tidy selected checker inventory failed") + enabled = check_names(Path(selected["stdout"]).read_text()) + if not required <= enabled: + raise ValueError("clang-tidy omitted required checks") + return {"version": text.strip(), "binary_sha256": digest(self.binary), "required_checks": sorted(required), + "enabled_checks": sorted(enabled), "metadata_commands": [version, catalog, selected]} + + def analyze(self, source, database, config_path, label, timeout): + started = time.monotonic() + report = self.output / (label + ".yaml") + profiles = self.output / (label + "-profiles") + profiles.mkdir() + argv = self.prefix + [str(source), "-p", str(database), "--config-file=" + str(config_path), + "--system-headers", "--enable-check-profile", "--store-check-profile=" + str(profiles), + "--extra-arg=--gcc-toolchain=" + str(self.sdk / "root/usr"), + "--extra-arg=-resource-dir=" + str(self.sdk / "root/usr/lib/llvm-14/lib/clang/14.0.6"), + "--export-fixes=" + str(report)] + result = execute(argv, self.workspace, self.output, label, timeout) + raw = Path(result["stdout"]).read_text() + Path(result["stderr"]).read_text() + frontend_failed = bool(re.search(r"clang-diagnostic-error|Error while processing|LLVM ERROR:|fatal error:|" + r"Error while trying to load a compilation database|Running without flags|Skipping .*file", raw)) + profile = consumer_profile(profiles, Path(source)) + names, diagnostics = [], [] + if report.is_file(): + names, diagnostics = diagnostic_report(report, source, raw, self.workspace, self.output, label, + timeout - (time.monotonic() - started)) + elif result["exit"] != 0 or frontend_failed or re.search(r":\d+:\d+: (?:warning|error):", raw): + raise ValueError("Missing clang-tidy diagnostic report: " + str(report)) + # Pinned Clang14 omits export-fixes entirely when there are no selected diagnostics. + result.update(source=str(source), report=str(report) if report.is_file() else None, diagnostic_ids=names, + diagnostics=diagnostics, consumer_profile=profile, + report_status="exported" if report.is_file() else "not-emitted-for-clean-input", + frontend_success=not frontend_failed and result["exit"] in (0, 1)) + if result["exit"] == 0 and (frontend_failed or names): + raise ValueError("clang-tidy returned success without a clean frontend/report") + return result + + +def controls(tool, units, build, config_path): + started = time.monotonic() + results = [] + for index, source in enumerate(units): + result = tool.analyze(source, build, config_path, "control-" + str(index + 1), 300 - (time.monotonic() - started)) + results.append(result) + if not result["frontend_success"]: + raise ValueError("Control frontend failed: " + str(source)) + if source.name == "raw_defects.cpp": + expected = {"clang-analyzer-unix.Malloc", "clang-analyzer-cplusplus.NewDeleteLeaks", + "clang-analyzer-cplusplus.NewDelete", "clang-analyzer-unix.MismatchedDeallocator", + "clang-analyzer-core.CallAndMessage", "bugprone-use-after-move", "clang-analyzer-cplusplus.Move"} + if result["exit"] != 1 or len(result["diagnostic_ids"]) != 7 or set(result["diagnostic_ids"]) != expected: + raise ValueError("Dirty control did not produce its exact seven required findings") + raw = Path(result["stdout"]).read_text() + markers = [(source, "expect-malloc", "clang-analyzer-unix.Malloc"), + (source, "expect-new-leak", "clang-analyzer-cplusplus.NewDeleteLeaks"), + (source, "expect-new-uaf", "clang-analyzer-cplusplus.NewDelete"), + (source, "expect-uninitialized", "clang-analyzer-core.CallAndMessage"), + (source, "expect-genuine-move", "bugprone-use-after-move"), + (source, "expect-genuine-move", "clang-analyzer-cplusplus.Move"), + (source.parent / "fixture_sdk/release.hpp", "expect-mismatch", "clang-analyzer-unix.MismatchedDeallocator")] + for path, marker, check in markers: + line = next(i + 1 for i, value in enumerate(path.read_text().splitlines()) if marker in value) + pattern = re.escape(str(path)) + ":" + str(line) + r":\d+: error: [^\n]*\[" + re.escape(check) + r"(?:,|\])" + if not re.search(pattern, raw): + raise ValueError("Control finding missing its independently marked location: " + marker) + if "allocation-owned" not in source.read_text() or not re.search(re.escape(str(source)) + r":\d+:\d+: note: Memory is allocated", raw): + raise ValueError("System-header mismatch lost its first-party allocation path") + elif result["exit"] != 0 or result["diagnostic_ids"]: + raise ValueError("Clean ownership/cancellation control produced diagnostics") + return results + + +def main(): + if len(sys.argv) not in (3, 4) or sys.argv[2] != "pi-release" or (len(sys.argv) == 4 and sys.argv[3] not in ("all", "controls", "check")): + raise ValueError("usage: analyze.py SDK pi-release [all|controls|check]") + mode = sys.argv[3] if len(sys.argv) == 4 else "all" + workspace = Path(__file__).resolve().parents[2] + sdk = Path(sys.argv[1]).resolve() + build = workspace / ".data/cpp-build/pi-release" + config_path = workspace / "cpp/.clang-tidy" + output = workspace / ".data/cpp-evidence" / ("analysis-" + str(time.time_ns())) + output.mkdir(parents=True, mode=0o700) + started = time.monotonic() + receipt = {"workspace": str(workspace), "preset": "pi-release", "mode": mode, "units": [], "status": "failed", + "runner_sha256": digest(__file__), "compiler_limit_sha256": digest(workspace / "cpp/tools/compiler-limit.py")} + code = 1 + try: + configuration = policy(config_path) + units = compilation_units(build / "compile_commands.json", workspace, build, sdk) + receipt.update(configuration=configuration, config_sha256=digest(config_path), + compile_database_sha256=digest(build / "compile_commands.json"), expected_units=list(map(str, units)), + source_sha256={str(path.relative_to(workspace)): digest(path) for path in units + sorted((workspace / "cpp/include").rglob("*.hpp"))}) + tool = Tool(sdk, workspace, output) + receipt["tool"] = tool.metadata(configuration, config_path) + baseline_path = workspace / "cpp/clang14-beast-read-baseline.json" + baseline = beast_baseline(baseline_path, sdk, workspace) + receipt["baseline_sha256"] = digest(baseline_path) + fixtures = [source for source in units if source.parent == workspace / "cpp/tests/analysis"] + real = [source for source in units if source not in fixtures] + receipt.update(expected_real_units=list(map(str, real)), expected_control_units=list(map(str, fixtures))) + if mode != "check": + receipt["controls"] = controls(tool, fixtures, build, config_path) + if mode != "controls": + phase_start = time.monotonic() + for index, source in enumerate(real): + result = tool.analyze(source, build, config_path, "tu-" + str(index + 1), 300 - (time.monotonic() - phase_start)) + receipt["units"].append(result) + if not result["frontend_success"]: + raise ValueError("Frontend failed for " + str(source)) + if mode != "controls": + classification = classify_beast(receipt["units"], baseline, sdk, workspace) + receipt["classification"] = classification + code = int(classification["new_count"] != 0 or classification["missing_count"] != 0 or classification["unclassified_exit_count"] != 0) + receipt["status"] = "passed_with_four_known_diagnostics" if code == 0 else "findings_or_stale_baseline" + else: + code, receipt["status"] = 0, "controls_passed" + except (OSError, ValueError, KeyError, IndexError, TypeError, TimeoutError, subprocess.SubprocessError) as error: + receipt["error"] = str(error) + finally: + receipt["duration_s"] = round(time.monotonic() - started, 6) + receipt["child_max_rss_kib"] = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + receipt["rss_scope"] = "RUSAGE_CHILDREN cumulative maximum for this serial invocation, not cgroup peak" + receipt["source_revision"] = subprocess.check_output(["git", "rev-parse", "HEAD", "HEAD^{tree}"], cwd=workspace, text=True).splitlines() + receipt["git_status"] = subprocess.check_output(["git", "status", "--porcelain"], cwd=workspace, text=True) + path = output / "receipt.json" + path.write_text(json.dumps(receipt, indent=2) + "\n") + classification = receipt.get("classification", {}) + print(json.dumps({"status": receipt["status"], "receipt": str(path), "units": len(receipt["units"]), + "classified": classification.get("classified_count"), "new": classification.get("new_count"), + "missing": classification.get("missing_count"), "error": receipt.get("error")})) + return code + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError) as error: + print(str(error), file=sys.stderr) + sys.exit(1) diff --git a/cpp/tools/build.py b/cpp/tools/build.py index 2889a693a..f53ae14d4 100644 --- a/cpp/tools/build.py +++ b/cpp/tools/build.py @@ -65,12 +65,12 @@ def allocated(paths): "test": [str(view / "bin/ctest"), "--preset", preset], } phases = ["configure", "build", "test"] if phase == "complete" else ["configure", "build"] if phase == "checkpoint" else ["configure", "test"] if phase == "test-checkpoint" else [phase] +if phase == "complete" and preset == "pi-release": + phases.append("tidy") receipts = [] for current in phases: if current == "tidy": - units = sorted((source / "src").glob("*.cpp")) - argv = [loader, "--library-path", libraries, str(sdk / "root/usr/bin/clang-tidy-14"), *map(str, units), "-p", str(build), "--checks=-*,clang-analyzer-core.*,clang-analyzer-cplusplus.*,bugprone-dangling-handle,bugprone-use-after-move", "--warnings-as-errors=*", "--extra-arg=--gcc-toolchain=" + str(sdk / "root/usr"), "--extra-arg=-resource-dir=" + str(sdk / "root/usr/lib/llvm-14/lib/clang/14.0.6")] - argv = ["/usr/bin/python3", str(source / "tools/compiler-limit.py"), *argv] + argv = ["/usr/bin/python3", str(source / "tools/analyze.py"), str(sdk), preset] else: argv = commands[current] stamp = str(time.time_ns()) diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 9b7e08b30..fa95fed5f 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -2672,3 +2672,11 @@ Research evidence and standalone CMake pilot changes use first-party import/lite The first C++ pilot is explicitly `stream: true` only, matching the qualified pinned Codex profile. False or missing stream requests refuse before provider dispatch with a standard OpenAI-shaped validation error and source-compatible authenticated reserve/release counts. Buffered TypeScript behavior remains unchanged and unsupported in this pilot; supporting it or claiming a general OpenAI replacement requires a separate accepted scope. The loopback C++ resource profile requests16KiB downstream `SO_SNDBUF` and records the effective kernel value. Its existing inherited fixture read-state exposes request/write/read counters so a namespace-local paused reader can prove pressure, stable upstream reads and eventual drain/cancellation. This is part of bounded transport acceptance, not a new public control surface. + +## Pinned C++ ownership analysis uses explicit coverage and defect controls - 2026-11-13 + +The requested GPT Pro recommendation selects the existing clang-tidy/Clang Static Analyzer 14.0.6 engine plus exactly `clang-analyzer-unix.Malloc` and `clang-analyzer-unix.MismatchedDeallocator`; existing core/C++ and dangling-handle/use-after-move checks remain fatal. The Pi complete workflow enforces the gate, with an explicit always-run CMake target. Its database contains all thirteen real production/test roots and three separately counted analysis-only controls. Checker membership, frontend status, raw diagnostics and a fresh source-matched AST-consumer profile must agree; neither an exit zero nor a profile alone proves successful or exhaustive analysis. Broad header reporting preserves emitted SDK endpoints and first-party caller paths without blanket vendor suppression. + +Representative known defects must produce their exact marked diagnostics, and clean ownership/cancellation-pattern controls must pass separately. Existing ASan/UBSan/leak and six real CTests remain required where relevant. Static heap checks do not prove descriptor lifetime, shared-owner cycles or coroutine/cancellation safety. Extra analyzers, alpha checks, annotation rollout and SDK changes are deferred; this tooling changes no runtime/provider behavior or production authority. See `cpp/docs/static-analysis.md` for commands, measured development evidence and pending canonical acceptance. + +The first complete scan's four pinned Beast `bugprone-use-after-move` findings remain preserved as a failed receipt. Source inspection and a real reduced static probe proved the Asio stackless yield/reentry path-sensitivity limitation, with equivalent explicit-return code clean and genuine same-invocation misuse fatal. A finite checked-in baseline may classify only those four exact Session-TU SDK fingerprints and move/loop notes under matching tool/frontend/header/policy identities, preserving raw exit1 and explicit classified/new counts. New, changed, missing or first-party findings and identity drift remain fatal; no checker, SDK directory or runtime source is suppressed or changed.