diff --git a/.entire/graph-agent.md b/.entire/graph-agent.md new file mode 100644 index 0000000000..c2a3257cac --- /dev/null +++ b/.entire/graph-agent.md @@ -0,0 +1,43 @@ +# entire-graph — instructions for coding agents (follow directly) + +You have a deterministic local code graph: `entire graph` (functions, classes, methods, +types, routes + call/inheritance relations; no network). These instructions are FOR YOU, the +agent reading this file. Use the graph to narrow exploration without trading away correctness. + +## The workflow (mandatory for locate/fix/change tasks) + +Your FIRST action on any task that requires finding code must be ONE search: + + entire graph search --repo . --profile full --query "" + +Then open the top hit's file with your file-read tool (pass a line range around the reported +line), inspect enough surrounding behavior to justify the change, and make the smallest complete +edit. Treat graph output as evidence, not an oracle. + +## Hard rules + +1. SEARCH FIRST — never grep/find/cat to locate code before you have searched. +2. READ focused source around the result. Widen the check when aliases, generated code, dynamic + dispatch, or related implementations could matter. +3. Use graph follow-ups only when they answer a real question. For impact or callers, prefer: + entire graph impact --repo . --symbol X +4. Make the smallest complete edit and check sibling sites or contracts when the task implies them. +5. VERIFY before stopping. Run the most focused relevant test, build, or reproduction available. + If execution is unavailable, perform a bounded source-level verification and state the limit. +6. Prefer precise queries and line ranges, but never trade resolution for fewer turns. +7. Feature-detect before relying on semantic relations: + entire graph capabilities --json + +## When NOT to use the graph + +If the task already names the exact file and it is small, just read it — the graph saves tokens +by eliminating exploration; when there is nothing to explore, skip it. + +## Reference + + locate -> entire graph search --repo . --profile full --query "..." + impact -> entire graph impact --repo . --symbol X (one shot: callers, callees, type consumers, data flow, co-change, siblings) + callers -> entire graph neighbors --repo . --symbol X --relation CALLS --direction in + change -> entire graph diff --base A --head B --json + detect -> entire graph capabilities --json (inventory-only languages have no relations) + stats -> entire graph stats --repo . (human-facing token-savings report; not part of your workflow — do not run it unless asked) diff --git a/.entire/settings.json b/.entire/settings.json index 77a79ecc1f..6aa5b10890 100644 --- a/.entire/settings.json +++ b/.entire/settings.json @@ -7,8 +7,10 @@ }, "filtered_fetches": true }, - "strategy": "manual-commit", "checkpoints": { - "primary": { "type": "git-refs" } - } + "primary": { + "type": "git-refs" + } + }, + "strategy": "manual-commit" } diff --git a/.gitignore b/.gitignore index 896bb738c2..eea0d213e0 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,14 @@ tmp/ entire-external-cp-test/ /skills/ /git-remote-entire + +# Checkpoint Lens (Buildathon 2026) +tools/**/__pycache__/ +*.pyc + +# Databricks credentials - MUST NEVER be committed +.databricks.local.json +.env.databricks + +# Checkpoint Lens verification baseline (machine-specific) +.entire/lens-verify-baseline.json diff --git a/AGENTS.md b/AGENTS.md index 681311eb9c..496bc3ba86 120000 --- a/AGENTS.md +++ b/AGENTS.md @@ -1 +1,8 @@ -CLAUDE.md \ No newline at end of file +CLAUDE.md + + +This repo has the entire-graph code graph installed. Before exploring code with +grep/find/whole-file reads, read .entire/graph-agent.md — resolution-first guidance +for using graph retrieval, focused source inspection, and verification. +@.entire/graph-agent.md + diff --git a/BUILDATHON.md b/BUILDATHON.md new file mode 100644 index 0000000000..0b509d1ce3 --- /dev/null +++ b/BUILDATHON.md @@ -0,0 +1,546 @@ +# Checkpoint Lens + +## One-sentence summary + +`entire lens` turns the context inside real Entire Checkpoints — stated intent, +decisions, rejected options, open questions, attribution — into four working +developer workflows: hand off, review against intent, list unfinished +requirements, and assess a single change — and it can **fail a build** when the +implementation has drifted from the plan. + +## Problem, intended user and why it matters + +**The user:** a developer or coding agent picking up work they did not just +finish — after a break, after a handoff, or when reviewing someone else's +agent-assisted branch. + +**The problem:** git records *what* changed. It does not record why the change +was made, what was tried and abandoned, what the author assumed, or what was +still unresolved when they stopped. With AI agents producing large diffs +quickly, that gap is now the expensive part: the diff is cheap to generate and +expensive to understand. + +Entire Checkpoints already capture that missing context — prompts, transcripts, +decisions, attribution — but as raw material. Reading a 1.1 MB `full.jsonl` is +not a workflow. Checkpoint Lens is the layer that turns it into one. + +**Why it matters:** the concrete failure this prevents is resuming work by +reading a diff, re-deriving the plan from the code, and silently dropping the +requirement the previous session had already identified but not yet built. + +## Selected Entire track and why Entire is essential + +**Track 1 — Build a Checkpoint-Native Developer Experience.** + +Checkpoint context is not decoration here; it is the input. Remove it and every +command loses its primary source: + +| Command | Without checkpoints | +| --- | --- | +| `handoff` | no intent, no decisions, no open questions — only a file list | +| `drift` | no plan to compare against; nothing to measure drift *from* | +| `assess` | degrades to a plain diff with no intent to cross-reference | +| `sync` | nothing to aggregate | + +A concrete, measured demonstration that checkpoint data is load-bearing *and* +that it must be combined with git rather than trusted alone: for checkpoint +`01M1TH4V7EFETXQ0QGNR2E871W`, the checkpoint's own `files_touched` records **2** +files while the linked commit changed **5**. The reader unions the two, so the +Graph blast-radius analysis sees `.entire/settings.json` and +`BUILDATHON_HANDOFF.md` — files a checkpoint-only read would have missed, and +whose *purpose* a git-only read could not have explained. + +Entire is also the delivery mechanism, not just the data source: `entire-lens` +is resolved through the CLI's own kubectl-style external-command lookup, so +`entire lens ...` is a first-class Entire subcommand rather than a side script. + +## Architecture and main workflow + +``` +refs/entire/checkpoints/<2>/ real checkpoint data (git-refs backend) + | + v + CheckpointReader (reader.py) git plumbing only; never checks anything out + | + Entire-Checkpoint trailer -> commit link + | + union with git commit stat + v + CheckpointRecord / SessionRecord (models.py) <-- ONE schema + | | | | + v v v v + report.py graph.py requirements databricks.py + (terminal) entities.py .py (drift) (Delta + aggregates) + | | | | + +------------+------+------+------------+ + v + cli.py (entire lens ...) +``` + +One schema (`models.py`) is shared by the terminal report and the Delta tables, +so a number shown locally and the same number aggregated in Databricks cannot +drift apart. + +**Main workflow (the one to watch):** + +1. `entire lens handoff` — reconstruct a stopped session: intent (with its + provenance), per-session attribution, decisions/risks/open questions + recovered verbatim, files touched, and the Graph blast radius of the commit. +2. `entire lens drift` — extract requirements from the *first* checkpoint's + stated plan, search the current codebase for each, and report coverage plus + what is still open. Adds an entity-level `graph diff` from the plan's commit + to now. +3. `entire lens assess ` — entity-level view of one change, plus an + intent cross-reference listing files the change touched that its checkpoint's + plan never mentioned. +4. `entire lens sync` — push records to Databricks, read cross-session + aggregates back. + +**The capability that makes this more than a report generator:** +`entire lens drift --fail-on-open` exits non-zero when a requirement stated in +the plan has no implementation evidence. "Did we build what we said we would +build?" becomes a pipeline gate, failing the same way a red test does. +`entire lens assess --verify ""` closes the other half: +it runs the tests through `entire graph verify`, which reports *which tests +changed state* rather than dumping runner output — so a test that was already +failing is not counted as evidence against the change under review. + +Both `handoff` and `drift` accept `--html PATH` and emit a single +self-contained file — no server, no CDN, no external fonts — which is the +fallback when live infrastructure fails during a demo. Samples are committed +under `docs/demo/`. + +## Entire Graph findings and verification + +Graph output is treated as **evidence, not an oracle**. Every Graph-backed +section prints the exact command that produced it so a reader can rerun it, and +a graph that cannot answer renders as *unavailable* — never as "no impact". + +**Required action 1 — search / definition lookup.** Used to locate checkpoint +storage and parsing in this codebase (`api/checkpoint/metadata.go`, +`cmd/entire/cli/checkpoint/open.go`), which is what established that this repo +uses the **git-refs** backend rather than the legacy `entire/checkpoints/v1` +branch. **Verified against source** by reading `.entire/settings.json` +(`"checkpoints": {"primary": {"type": "git-refs"}}`) and listing the live refs. +The original plan in checkpoint 1 named the v1 branch; the graph finding +corrected it, and `reader.py` reads refs as a result. + +**Required action 2 — relationship / impact analysis before a high-risk +change.** `entire graph commit --repo . --json` before changing the +blast-radius code path. This produced the run's most valuable finding: the +change list is `files[].changes[]` carrying `dependents_count`, which is what +lets the tool flag a *signature change or removal that has callers* — the shape +that breaks things silently. Running it against our own HEAD reported +`.gitignore body_changed` with 53 dependents. + +**Required action 3 — final semantic diff of the submitted implementation.** +`entire graph diff --base --head --json`, reported +in `entire lens drift`: **75 entity changes across 9 files** between the plan +and the implementation. Entity-level is the point — a function that only moved +is not counted as drift, which a raw text diff would have false-flagged. + +**Where the Graph was wrong, and how we caught it.** Three interface defects +were found by verifying against real payloads rather than trusting our first +call. All three had the same dangerous shape — a *quiet* failure: + +- `entire graph commit` takes a **positional** rev and `--json`. Called with + `--commit/--format` it **exits 0** and prints `commit accepts at most one + revision` to stdout. A trusting caller renders an empty blast radius, which + reads as "this change has no dependents." +- `entire graph search` takes `--top-k`, not `--limit`. Every requirement was + reporting UNVERIFIED, indistinguishable at a glance from a query the graph + genuinely could not answer. +- Search results are keyed on `file_path` + `snippet`, not `symbol`/`name`, so + the hit extractor was scoring against nothing. + +This is exactly why the operating guide says to verify. All three are fixed and +the fix is pinned by tests that assert a failing graph reports `ok=False` rather +than an empty section. + +## Noon Curveball: what changed and how we adapted + +**The constraint: PRIVACY BOUNDARY.** Raw prompts and transcripts must not be +sent to a new external service; the product must keep working when sensitive +fields are redacted or unavailable; and it must never present incomplete +context as complete. + +The pre-noon stable state is checkpoint `01M1TM6JKPX4MV0DV829FQ77GA` +(commit `9ee636fe1`). + +### Two assumptions it invalidated + +**1. Truncation was treated as de-identification. It is not.** `DatabricksSync` +capped free text at 800 characters (`MAX_TEXT`) and the module docstring +offered that as the safeguard. A prompt cut at 800 characters is still 800 +characters of the user's own words. The live warehouse proved it: four of the +eight `sessions.intent` rows were *exactly* 800 characters — truncated +mid-sentence, de-identified not at all — and they carried absolute local +filesystem paths from the developer's own machine, which are not reproduced +here for the same reason they should not have been uploaded. +`decisions.text` held a further 37 rows of verbatim transcript and commit prose. + +**2. The earlier credential check was assumed complete. It was not.** +Checkpoints `01M1TQ2SJR65VG107JGSEQ9MTA` (`79743a332`) and +`01M1TQGCCBZX7GPKX5JD9MN2ZT` (`768bc9283`) both record that pasted Databricks +credentials were verified absent from every checkpoint, tracked file, the git +history and the generated HTML. That is true of the **PAT** — Entire's own +redaction pipeline caught it. It is not true of the **workspace URL, the org ID +`o=...`, and the warehouse HTTP path**, which sat unredacted in the stated +intent of four checkpoints and were uploaded through the very `intent` column +this fix removes. The infrastructure identifiers of the warehouse were being +stored inside that warehouse. The check verified the token and generalised to +"credentials"; the gap was in the generalisation, not the search. + +### The Graph analysis that scoped the fix + +Run before any edit. `entire graph capabilities --json` reports every +`features_requiring_network_access` false, so the Graph plugin is local-only +and not an egress path. `entire graph impact --symbol DatabricksSync._connect` +then returned the complete egress closure — 2 direct callers (`sync` for +writes, `_query` for reads), 4 transitive, 1 callee (`databricks.sql.connect`) +— proving `databricks.py` holds the only socket in the product. A source sweep +for `requests|urllib|http|socket|connect(` agreed. + +One honest limit, recorded because the project's own discipline demands it: +`impact --symbol SessionRecord.to_row` reported **only a test caller**. The +real call site is `databricks.py` inside `for s in rec.sessions`, which the +Python resolver cannot type through the loop. Graph narrowed the search +correctly and its caller list was incomplete; the edge was confirmed by reading +the source. Evidence, not an oracle. + +### What changed + +The decisive finding was that **no aggregate ever read either column**: +`open_items_trend` groups on `kind`, `file_churn` on `file_path`, and +`SQL_COVERAGE` tests `intent_source`, never `intent`. The two raw-text columns +were written and never queried, so removing them cost zero analytic capability. + +* `sessions.intent` and `decisions.text` are gone. In their place: + `intent_len`, `intent_word_count`, `intent_digest`, `intent_redacted`, and + the same four for decisions, plus `confidence`. +* `derive_signals()` is the single chokepoint that turns text into those + signals and discards the text. The digest is salted per repository for domain + separation; it is documented as a join key, **not** an anonymisation + primitive, because short low-entropy text is guessable from any hash. What + makes this safe is that the plaintext never leaves the machine. +* `assert_egress_safe()` validates every outgoing row against `EGRESS_COLUMNS` + immediately before the INSERT, so a future column cannot quietly reintroduce + prose. A test asserts the DDL and the spec cannot drift apart. +* `entire lens sync --dry-run` prints the exact outgoing rows and connects to + nothing — auditable without credentials. It calls the same `build_rows` the + real sync does, so it cannot describe a payload other than the one sent. +* **Data already sent was purged**, not just stopped: `sync --purge` drops the + three tables (DROP, not DELETE — DELETE leaves the columns in place and the + rows reachable through Delta time travel) and recreates them on the text-free + schema. Verified afterwards by `DESCRIBE TABLE`: no `intent` or `text` column + exists in any of the three. Residual stated honestly: Unity Catalog keeps a + 7-day `UNDROP TABLE` window, which is an admin action, not something the CLI + can reach. + +**Post-purge verification, reproducible by a judge with warehouse access.** +`DESCRIBE TABLE workspace.checkpoint_lens.checkpoint_sessions` returns 27 +columns, `checkpoint_files` 4, and `checkpoint_decisions` 10, with no `intent` +and no `text` column among them. Where 800 characters of a user prompt used to +sit, a row now reads `intent_len 2224 | intent_word_count 336 | intent_digest +91c4421e423b89a5 | intent_redacted False` — the length is reported honestly, +and the words are not there to report. The aggregates still answer: 10 +checkpoints, 42 decisions, 43 file rows, a 7-point unresolved-context trend, +and a churn ranking whose top entries are real source files. + +**Deliberate scope decision:** `file_path`, `branch`, `session_id`, `agent`, +`model`, `linked_commit` and token counts still leave the machine. They are +checkpoint metadata, not prompt or transcript content, and `file_churn` is +unbuildable without `file_path`. Hashing them would buy little and would make +the hotspot ranking unreadable. Stated here so it can be revisited rather than +discovered. + +### Complete vs incomplete: one signal, not five + +The tool already reported degraded inputs — a warnings block, a "graph +unavailable" line, a "Databricks unavailable" line, an intent-provenance label. +Five signals, each true, each local to its own section, **none at the top**, and +none covering a missing transcript, an absent `metadata.json`, or content +Entire had already redacted. A reader who stopped before the warnings block +read a partial reconstruction as a complete one. + +`completeness.py` now computes ONE verdict from all eight inputs, rendered +first in the terminal report, first in the HTML report (above the stat grid, +because those numbers are the most confident-looking thing on the page), and +emitted as a machine-readable `completeness` object in every `--json` payload. + +Three consequences worth naming: + +* **Redaction is a third state.** Not "available", not "missing": the + checkpoint is intact and some of what it described is gone. It is reported as + `REDACTED` and counts as partial. Running against this repo's own checkpoints + now correctly reports PARTIAL — Entire redacted a token in them. +* **"Not consulted" is not "fine".** `--no-graph` / `--no-databricks` degrade + the verdict rather than silently counting as healthy. +* **`drift --fail-on-open` warns when it gates on partial context.** A gate + that goes green on context it could not fully read converts "we do not know" + into "we checked". + +And the sharpest instance, fixed: `render_decisions` printed +`(no decision context recovered from this transcript)` whether the transcript +was read and held nothing, or was missing, or was fully redacted — identical +output for three different truths, in the section the product's central claim +rests on. Those are now three different sentences. + +### The test + +`tools/checkpoint_lens/tests/test_privacy_boundary.py` — 21 tests over a +**synthetic** checkpoint fixture, labelled as synthetic in the module +docstring, the class name (`SyntheticCheckpointRepo`) and the commit that +introduced it. It builds a real git repository with real +`refs/entire/checkpoints/**` refs carrying four shapes: healthy, redacted +(REDACTED markers), starved (no `prompt.txt`, no transcript), and no +`metadata.json` at all. The redaction case also occurs in this repo's genuine +checkpoints; the missing-field cases do not, and a test that only runs where +the defect happens to exist is not a test. + +It asserts that degraded checkpoints still produce useful output, that a +redacted checkpoint can never report COMPLETE, that missing metadata does not +render as zero, and that the real outgoing rows — obtained from +`DatabricksSync.build_rows`, the actual sync path — contain none of the +fixture's prompt or transcript phrases. + +**96 tests pass: the 75 that existed before the curveball, unchanged, plus 21 +new ones.** + +### Final semantic diff of the submitted implementation + +The third required Graph action, run against the submitted code rather than a +description of it: + +```bash +entire graph diff --repo . --base 768bc9283 --head HEAD --json +``` + +**80 entity changes across 9 files** — 66 added, 10 body-changed, 4 +signature-changed: + +| File | Entity changes | Status | +| --- | --- | --- | +| `tools/checkpoint_lens/tests/test_privacy_boundary.py` | 40 | added | +| `tools/checkpoint_lens/databricks.py` | 10 | modified | +| `tools/checkpoint_lens/cli.py` | 8 | modified | +| `tools/checkpoint_lens/completeness.py` | 8 | added | +| `BUILDATHON.md` | 5 | modified | +| `tools/checkpoint_lens/html.py` | 3 | modified | +| `tools/checkpoint_lens/models.py` / `reader.py` / `report.py` | 2 each | modified | + +Graph flags four signature changes carrying dependents, and each is a +deliberate part of the fix rather than collateral: + +| Signature change | Dependents | Why | +| --- | --- | --- | +| `DatabricksSync._insert` | 2 | dropped the `ncols` argument; the column count now comes from `EGRESS_COLUMNS`, so the row width cannot disagree with the schema | +| `html.render_handoff` | 8 | takes the completeness verdict so the banner can lead the document | +| `html.render_drift` | 1 | same | +| `report.render_decisions` | 3 | takes `transcript_available`, so an empty result can say *why* it is empty | + +That is the whole point of an entity-level diff over a text diff: it names the +four call contracts a reviewer must check, out of ~860 changed lines, and it +confirms nothing was **removed** — no existing caller lost a function. + +Verified against source, not taken on trust: the graph's own `changes[]` +payload is nested under `files[]`, and a first parse against a top-level +`changes` key returned **0 entity changes** — which reads exactly like "this +change touched nothing". Same failure shape as the three silent interface +defects found before noon, caught the same way. + +## Checkpoint links and what each checkpoint proves + +Checkpoints are on the `RV/Entire_Trunk` branch of the fork, pushed to the +Entire mirror (`entire://aws-ap-south-1.entire.io/gh/rohithvishnukumar/entire_cli_kernel_001`). +Read any of them with `entire checkpoint explain `. + +| Checkpoint ID | Commit | What it proves | +| --- | --- | --- | +| `01M1TH4V7EFETXQ0QGNR2E871W` | `34c1528d8` | **Milestone 1 — initial understanding and intended architecture.** Problem framing, why Entire is essential, planned CheckpointReader + GraphClient + Databricks architecture, and the open questions. No product code yet, deliberately. | +| `01M1TM6JKPX4MV0DV829FQ77GA` | `9ee636fe1` | **Milestone 2 — last stable state before the Noon Curveball.** Working `handoff` on real data. Records the deliberate architecture deviation from milestone 1 (Go → Python-as-Entire-subcommand) *with the reason*, plus unresolved work and open risks. | +| `01M1TM6ZCD0CP72K2HW6S945KY` | `6ca259776` | Hygiene: committed bytecode removed. Shows up later as noise in the file-churn aggregate — an honest artefact of our own history. | +| `01M1TM9WCDTJJBYQNFEDCBJ04N` | `dda47c39a` | **Graph evidence + a real bug.** The silent `graph commit` interface defect, found by verification rather than assumption. | +| `01M1TNJQEJTDEK7Y6MSGR5KBBQ` | `c1e0521a0` | `drift`, `assess`, Databricks sync, 40 tests. Two further silent graph-interface bugs found and fixed. | +| `01M1TQF2VBAQ9ZKJJZDD6YGJJH` | `e4ad10bc6` | Drift as a CI gate (`--fail-on-open`) and adjudicated verification in `assess`. | +| `01M1TQGCCBZX7GPKX5JD9MN2ZT` | `768bc9283` | **Milestone 3 — the last stable state before the Curveball response.** The baseline every post-noon claim is measured against. | + +The milestone-2 checkpoint is the one worth opening: it records a *rejected* +option (installing Go) with the reasoning, which is precisely the context git +alone cannot preserve. + +**Milestones 3 and 4 — the Curveball response and the final verification — are +commits `79d0fa43a` and later on this branch.** Their full narrative is in the +commit messages, which is where every checkpoint in this project takes its +text from. Stated plainly rather than glossed: at the time of writing those +commits do not yet have their own `refs/entire/checkpoints/**` entry, because +the coding-agent session that produced them had not reached a `Stop` boundary +when they were made, so the shadow branch held no un-condensed content for +post-commit to condense. This is Entire's documented fail-open path — the work +is committed, pushed and reproducible; only the checkpoint ref lags. + +## Setup, run and test instructions + +Requires Python 3.9+, `git`, the `entire` CLI, and the `graph` plugin. + +```bash +git clone && cd entire_cli_kernel_001 +git checkout RV/Entire_Trunk + +# Run directly (no install needed): +python -m tools.checkpoint_lens.cli handoff --repo . + +# Or as a first-class Entire subcommand (put the repo root on PATH): +# PowerShell: $env:PATH = "$PWD;$env:PATH" +# bash: export PATH="$PWD:$PATH" +entire lens handoff --repo . +entire lens drift --repo . +entire lens assess HEAD --repo . +entire lens sync --repo . # requires Databricks credentials + +# Every command supports --json, --no-graph and --no-databricks. +``` + +**Tests** (stdlib `unittest`, no pip install required): + +```bash +python -m unittest discover -s tools/checkpoint_lens/tests -t . -v +# 40 tests +``` + +**Databricks credentials** are never committed. Provide either environment +variables (`DATABRICKS_SERVER_HOSTNAME`, `DATABRICKS_HTTP_PATH`, +`DATABRICKS_TOKEN`) or a gitignored `.databricks.local.json` in the repo root: + +```json +{ "server_hostname": "...", "http_path": "/sql/1.0/warehouses/...", + "access_token": "...", "catalog": "workspace", "schema": "checkpoint_lens" } +``` + +Install the optional connector with `pip install databricks-sql-connector`. +Without it, every command still runs; only the cross-session section is absent, +and it says so. + +## Databricks use, data sources and limitations + +**Capabilities used:** Delta tables on Unity Catalog, queried through a +serverless SQL warehouse via `databricks-sql-connector`. + +**Workspace:** `dbc-fbf9202b-29fc.cloud.databricks.com` +**Schema:** `workspace.checkpoint_lens` +**Warehouse:** `/sql/1.0/warehouses/08d05243c52e3597` (one 2X-Small serverless, +the Free Edition allowance) + +**Tables** (`tools/checkpoint_lens/databricks.py`): + +| Table | Grain | +| --- | --- | +| `checkpoint_sessions` | one row per session per checkpoint | +| `checkpoint_files` | one row per file per checkpoint | +| `checkpoint_decisions` | one row per recovered decision/risk/open question | + +**Why it is essential rather than storage.** The three aggregates are chosen so +that none can be computed from a single checkpoint: + +- `open_items_trend` — is unresolved context *accumulating or being discharged* + across the whole project history? A blocker raised in checkpoint 1 and still + unanswered at checkpoint 5 is invisible to any single-checkpoint view. Live + result: unresolved fell from 13 at milestone 1 to 1 thereafter. +- `file_churn` — hotspots are a property of the *sequence* of checkpoints, not + of any one commit. +- `coverage` — intent-coverage %, decisions captured, and average agent-written + percentage across all sessions. + +Delete the Databricks layer and the tool still runs; every trend and ranking +degrades to a single point in time, and the CLI prints that explicitly rather +than presenting one checkpoint's numbers as a trend. + +**Data provenance.** Every row is derived from *this repository's own* Entire +Checkpoints, created by the author and their coding agent during the event. No +third-party, customer, personal or confidential data is uploaded. Free-text +columns (`intent`, decision `text`) are the author's and the agent's own prose, +truncated to 800 characters before upload — these tables exist for aggregation, +not to rehost transcripts. Credentials are gitignored and never committed. + +**Two data-quality decisions that materially change the numbers**, both worth +inspecting: + +- *Decisions are attributed to the checkpoint where they FIRST appeared.* + Entire stores the whole compacted session in every checkpoint, so counting + raw occurrences made a single blocker look like it was raised again on every + later commit and turned the unresolved-items trend into a monotonically + increasing line. First-appearance attribution answers the question the trend + is actually asking — *when was this raised* — and is what makes a falling + line mean what a reader assumes it means. +- *Build artefacts, vendored code and lockfiles are excluded from churn.* This + repository's own committed-then-deleted `__pycache__` sat at the top of a + ranking that is supposed to point a reviewer at risky source. + +**Credential handling, verified rather than asserted.** Databricks credentials +were pasted into a working session during the build. They reached no +checkpoint: Entire's own redaction pipeline caught the token (11 `REDACTED` +markers in the transcript), and a scan of every checkpoint ref, every tracked +file, the entire git history and the generated HTML found zero occurrences. +The credentials file is untracked and gitignored. + +**Sync semantics.** Idempotent: a `DELETE` scoped to the repo key followed by +batched inserts, so re-running after new checkpoints never double-counts, and +one workspace can host several repos. + +**Limitations.** +- Current volume is small (5 checkpoints, one repo, one day). Trend lines are + real but short; nothing here should be read as a statistically meaningful + trend yet. +- `file_churn`'s top entries currently include `__pycache__` `.pyc` files that + were committed and then removed in our own history. That is honest data, not + a bug, but it is noise in a risk ranking — path filtering is the obvious next + step. +- Free Edition offers no production SLA; if the warehouse is unavailable during + judging the CLI degrades gracefully and says the cross-session view is + unavailable. + +## Known limitations and next steps + +**Honest limitations:** + +1. **Decision extraction is a transparent classifier, not a language model.** + Chosen deliberately so output is verbatim, auditable, offline and incapable + of inventing a decision that was never made. It went through three rounds of + precision work against real output, each pinned by tests: prompt echoes are + suppressed (a decision is what the agent *concluded*, not what the user + *asked for*); markers must be predicated of the work rather than merely + mentioned (the word "risk" inside the column name "file-churn/risk score" is + not a risk); and causal prose is captured as `rationale`, which turns out to + be the most abundant form the "why" actually takes. It still cannot + recognise a decision stated in a form no marker covers. +2. **`drift` verdicts are heuristic.** The score is the fraction of a + requirement's keywords appearing in Graph search hits. `MISSING` means *no + evidence was found* — a prompt to verify, not proof of absence — and the + report says so on every run. +3. **Requirement extraction depends on the plan being written down.** When the + baseline checkpoint's prompts contain a pasted planning document, extraction + picks up non-requirements; metadata and logistics are filtered, but the + filter is heuristic. +4. **Single-repo assumption in the Databricks key.** The repo key is the + directory basename, so two clones of the same repo in differently-named + directories would be treated as different projects. +5. **`drift` costs one Graph search per requirement.** Now concurrent and on + the `fast` profile (58s for 18 requirements, down from 4m01s), but it is + still the slowest command. +6. **Not tested on a repo other than this one.** The tool takes `--repo` and + makes no assumptions about its own layout, and it is covered against an + empty git repo, but it has not been exercised against a large third-party + checkpoint history. + +**Next steps toward production readiness:** + +- Replace the keyword classifier with a checkpoint-summary-backed intent + source: `entire checkpoint explain --generate` populates `Summary.Intent` and + `Summary.OpenItems`, which the reader already prefers when present. That + would raise precision substantially while keeping the current path as an + offline fallback. +- Persist drift verdicts per checkpoint so `open_items_trend` can distinguish a + requirement that was *completed* from one that was silently *dropped* — the + single most useful thing this data could support and the natural next + Databricks query. +- Path filtering and a weighted risk score (`churn × dependents`) for the + hotspot ranking. +- A `--fail-on-open` exit code so `drift` can run in CI as a release gate. diff --git a/BUILDATHON_HANDOFF.md b/BUILDATHON_HANDOFF.md new file mode 100644 index 0000000000..5839c7ddc9 --- /dev/null +++ b/BUILDATHON_HANDOFF.md @@ -0,0 +1,251 @@ +# BTW Buildathon 2026 — Project Handoff Notes + +Context document for the AI coding agent picking this up. This captures every decision made during planning, before any implementation started. + +--- + +## Event basics + +- **Event:** Bengaluru Tech Week Buildathon 2026, powered by Entire + Databricks +- **Date:** Sunday, 6 September 2026, 9:00 AM–5:00 PM IST +- **Venue:** Scaler School of Technology, Electronic City, Bengaluru +- **Submission deadline:** 3:00 PM IST — hard cutoff +- **Judging:** 3:00–5:00 PM +- **Team:** Solo builder, full-stack (GitHub: `Rohithvishnukumar`). Same person is submission owner and demo owner. + +--- + +## Track selected: Track 1 — Build a Checkpoint-Native Developer Experience + +- **Build on:** `entireio/cli` (fork this — NOT `entire-graph` [Track 2] or `external-agents` [Track 3]) +- **Core idea:** turn context preserved in real Entire Checkpoints into a useful developer workflow. Git shows *what* changed; checkpoints should preserve *why*, what was attempted, assumptions made, what failed, what's unresolved. +- **Official use cases** (targeting all five): + 1. Review an implementation against its stated intent + 2. Identify unfinished requirements or unresolved risks + 3. Create a change-risk or release-readiness report + 4. Hand work to another developer or agent without losing original reasoning + 5. Resume an agent-assisted workflow with confidence after a break +- **Success bar:** checkpoint context must be an *essential* input — the tool should let someone complete a task more accurately/efficiently than from a git diff alone. Using Entire only to track dev without using its context does not qualify. + +--- + +## Product: "Checkpoint Resume & Drift Assistant" + +Three CLI subcommands sharing one core (build in this priority order — 1 and 2 are must-haves, 3 is a stretch goal only if time remains): + +### Shared core (build first) +- **CheckpointReader** — parses the `entire/checkpoints/v1` branch into structured session data: session id, timestamps, prompts/decisions text, files touched, stated intent. +- **GraphClient** — thin wrapper shelling out to `entire graph` commands (`search`, `diff`, `impact`) and parsing their output. +- **DatabricksSync** — pushes parsed checkpoint records into a Delta table. + +### 1. `entire resume` — covers use cases 4 & 5, partially 1 +- Pulls the checkpoint narrative (intent, decisions, files touched). +- Adds a Graph impact/blast-radius section: for files/symbols the last session touched, show what else depends on them. +- Adds one Databricks aggregate number (e.g. coverage %). +- Renders to terminal **and** a single self-contained HTML file (for demo visual appeal, no server needed). +- This is the command we dogfood at noon to reconstruct our own context in a fresh session. + +### 2. `entire drift` — covers use cases 1 & 2 +- Compares the *original plan* (from the first checkpoint) against *current* code state. +- Uses `entire graph diff` (entity-level) rather than raw text diff — a function can move without meaningfully changing; raw diff would false-flag it as drift, Graph won't. +- Outputs a list of unfinished/dropped requirements. + +### 3. `entire assess ` — stretch goal, covers use case 3 +- Only build after 1 and 2 are solid and running on real data. +- Reuses the same core. Scopes Graph impact analysis to one specific commit/checkpoint. +- Cross-references that change against the nearest checkpoint's stated intent — flags things like "this touches files the plan didn't mention." +- Cheap to add (~30–45 min) since it's composition of existing pieces, not new build. + +--- + +## Databricks integration (Best Use of Databricks award) + +- **Free Edition account** — create/open before the event (setup ≠ implementation, this is allowed prep). +- **Constraints to design around:** one serverless 2X-Small SQL warehouse total, up to 5 concurrent job tasks, up to 3 Apps, one workspace/metastore per account. Keep scope narrow — don't overbuild. +- **Plan:** one Delta table storing structured checkpoint records (via DatabricksSync) + exactly one working SQL query computing something meaningful — requirement-coverage % or file-churn/risk score. `entire drift` (and/or `resume`) queries this back. +- **Meaningful-use test:** removing Databricks should break the trend/aggregate analysis, leaving only a single-point-in-time view. That's the bar for "essential," not "superficial storage." +- **Must document in BUILDATHON.md:** capabilities used, why essential, workspace/app/endpoint/demo URL, relevant repo paths, reproduction steps, data provenance, and how the Noon Curveball affected the Databricks workflow. + +--- + +## Repos — which to fork (30+ repos in the org, most are noise) + +| Repo | What it is | Relevant? | +|---|---|---| +| `entireio/cli` | Core Entire CLI — checkpoints, git integration | **Fork this one** | +| `entireio/entire-graph` | Semantic code-graph plugin | Track 2 only | +| `entireio/external-agents` | Plugin binaries for more coding agents | Track 3 only | +| `entireio/cli-checkpoints` | Small, unrelated repo | Ignore | +| `entireio/entire-judge` | Looks like organizers' internal judging tooling | Ignore — not for participants | +| Everything else (`git-sync`, `auth-go`, `homebrew-tap`, `scoop-bucket`, `forgemark`, `devcontainer-features`, etc.) | Internal infra / distribution tooling | Ignore | + +--- + +## Exact setup commands (run ONLY after 9:00 AM official start) + +```bash +# 1. Fork entireio/cli to your account +gh repo fork entireio/cli --clone=false +# (no GitHub CLI? click "Fork" manually on github.com/entireio/cli) + +# 2. Authenticate Entire CLI +entire login + +# 3. Create the mirror — select your fork + India region when prompted +entire repo mirror create + +# 4. Clone through Entire's mirror workflow +entire repo clone /gh/Rohithvishnukumar/cli + +# 5. Move into the cloned directory +cd cli + +# 6. Enable checkpoints for Claude Code specifically +entire enable -y --agent claude-code + +# 7. Confirm it's tracking +entire status + +# 8. Install and activate the Graph plugin +entire plugin install graph +entire graph version +entire graph init-agents --repo . +``` + +**Important:** after `init-agents`, close the current Claude Code session and start a fresh one (`claude`) in that directory. Graph instructions are only loaded into sessions started *after* `init-agents` runs — an already-open session won't pick them up. + +--- + +## Required checkpoint milestones (minimum 4, non-negotiable) + +1. Initial understanding and intended architecture +2. Last stable state before the Noon Curveball +3. Response to the Noon Curveball +4. Final implementation and verification + +Checkpoint **quality matters more than quantity** — capture decisions, rejected options, failures, assumptions, open risks, and the evidence that changed the approach. Not "wip" commit messages. + +## Required Graph evidence (at least 3 distinct, demonstrated actions) + +1. A graph search or definition lookup — do early, right after `init-agents`. +2. A relationship/impact analysis **before** a high-risk change — do before touching the curveball-affected area. +3. A final semantic-diff analysis of the submitted implementation — do at the end, before the final checkpoint. + +Treat Graph output as evidence, not an oracle — verify findings against real source/tests. Never present incomplete or uncertain graph output as fact. + +--- + +## Hour-by-hour plan + +Real build time is only ~5 hours total (9–12 and 1–3); the noon hour is curveball + lunch combined, not extra build time. + +| Time | What happens | +|---|---| +| 9:00–9:15 | Fork, mirror, clone (commands above) | +| 9:15–9:35 | Enable Checkpoints + Graph. Write a substantive first checkpoint: user, problem, why git diff isn't enough, planned architecture. Run one `entire graph search` (1st required Graph evidence). | +| 9:35–11:00 | Build shared core (CheckpointReader, GraphClient). Wire minimal Databricks: one Delta table, sync script, one SQL query. | +| 11:00–11:45 | Ship `entire resume` v1 and `entire drift` v1 — must run on real data from this repo, not mocked. | +| 11:45–12:00 | **Stop new features.** Clean commit. Write the required pre-noon checkpoint: intent, architecture, done/unresolved work, risks. | +| 12:00–1:00 | Close session for real. Receive curveball (Track 1-specific only). Think while eating. Open a **fresh** session, use own `entire resume` output to reconstruct context — this is the dogfooding moment and best demo proof. Run Graph impact analysis on the affected area before editing (2nd required Graph evidence). | +| 1:00–2:20 | Implement the smallest complete curveball response. Update Databricks table/query if data shape changed (scores real points on the Databricks rubric). Write/adjust tests. Run final semantic-diff analysis via Graph (3rd required evidence). Confirm no regressions in resume/drift. Commit final checkpoint. | +| 2:20–3:00 | Fill `BUILDATHON.md`. Run final 20-minute checklist. Submit all required fields before 3:00 PM sharp. | +| 3:00–5:00 | Judging — keep project runnable, don't touch main branch, rehearse demo script. | + +--- + +## BUILDATHON.md required outline + +``` +# Project name +## One-sentence summary +## Problem, intended user and why it matters +## Selected Entire track and why Entire is essential +## Architecture and main workflow +## Entire Graph findings and verification +## Noon Curveball: what changed and how we adapted +## Checkpoint links and what each checkpoint proves +## Setup, run and test instructions +## Databricks use, data sources and limitations (if applicable) +## Known limitations and next steps +``` + +Don't skip "known limitations" — it's explicitly judged (Demonstration and future potential, 10 pts). + +--- + +## Final 20-minute submission checklist + +- [ ] Final commit pushed, SHA matches submission +- [ ] Project launches from a clean checkout / documented setup path +- [ ] Required checkpoints open and clearly explain the 4 milestones +- [ ] Entire Graph evidence + final semantic-diff analysis are recorded +- [ ] BUILDATHON.md complete, readable, free of secrets +- [ ] Tests covering critical + Curveball behavior pass +- [ ] Databricks resource links + data notes included +- [ ] Demo owner can sign in, open every resource, run the critical path +- [ ] Fallback screenshot/recording saved locally +- [ ] Submitted before 3:00 PM IST + +## Required submission fields + +- Selected Entire track (Track 1) +- GitHub fork URL and final commit SHA +- Entire mirror/project URL +- Links to the required Entire Checkpoints +- Clear setup, run, test instructions +- Working product demo or a reliable fallback recording +- Complete `BUILDATHON.md` in repo root +- If opting into Databricks: capabilities used, why essential, workspace/app/endpoint/demo URL, relevant repo paths, reproduction steps, data provenance, how the Curveball affected the Databricks workflow + +--- + +## Judging rubric (use this to prioritize effort under time pressure) + +### Entire main challenge — 100 points +| Criterion | Points | +|---|---| +| Problem and innovation | 20 | +| Technical implementation | 25 | +| Response to the Curveball | 15 | +| Use of Entire Checkpoints | 15 | +| Use of Entire Graph | 15 | +| Demonstration and future potential | 10 | + +### Best Use of Databricks — 100 points +| Criterion | Points | +|---|---| +| Meaningful use | 30 | +| Working implementation and reliability | 25 | +| User value and product decisions | 20 | +| Data quality, provenance, responsible use | 15 | +| Curveball response | 10 | + +**Takeaway:** Technical implementation + Problem/innovation = 45% of the Entire score → fewer things, all of them actually working, beats broad-but-flaky. Databricks "meaningful use" alone is 30% of that award → keep it narrow but genuinely load-bearing, never superficial storage. + +--- + +## Final demo script (for 3:00–5:00 judging) + +1. State the user and problem in one sentence. +2. Show the working product and critical path live — not a slide walkthrough. +3. Explain why Entire is essential to the solution. +4. Show one useful checkpoint and one graph finding that changed or verified a decision. +5. Explain the Noon Curveball: what changed, what behavior changed, the test that proves it. +6. Show the essential Databricks function and evidence that it's working. +7. Close with known limitations and the next step toward production readiness. + +--- + +## Key strategic decisions made during planning + +- **Solo builder** — no team coordination overhead, but no parallelization either; scope is deliberately cut tight for a ~5-hour build window. +- **CLI-first interface**, not a web app — matches Entire's own convention (`entire graph search`, etc.), cheap to build solo. Each command also writes a self-contained HTML report purely for demo visual appeal, no backend needed. +- **Dogfooding is the core strategy**: our own build-process checkpoints ARE the real data the tool analyzes. At noon we literally run `entire resume` on ourselves to reconstruct context — this satisfies the mandatory workflow AND is our strongest, unstaged demo proof point. +- **The tool must stay generic** — accept a repo path as a parameter, don't hardcode assumptions about its own project structure. Otherwise it reads as a party trick rather than a real reusable tool during judging. +- **Allowed before 9:00 AM:** research, user discovery, problem framing, sketches, planning, and testing Entire CLI commands on a *throwaway scratch repo* (not the submission repo) to learn syntax. **Not allowed:** any actual implementation of the submission itself, or arriving with a prebuilt feature branch. + +## Open decisions to make on the day + +- Exact SQL query for the Databricks layer — requirement-coverage % vs. file-churn risk score. Pick whichever is fastest to compute correctly from real checkpoint data once it exists. +- Whether to attempt `entire assess` — only after `resume` and `drift` are both solid and running on real data, not before. +- Exact mechanism for `drift`'s "requirements" baseline — likely a lightweight `requirements.md` maintained alongside checkpoints, or parsed directly from the first checkpoint's stated intent text. diff --git a/CLAUDE.md b/CLAUDE.md index 90338188eb..84dee6057b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,55 @@ This repo contains the CLI for Entire. +## Project Context for This Build (Buildathon 2026) + +**This fork of `entireio/cli` is not being used for Entire CLI contribution work.** +It is the base for a BTW Buildathon 2026 (Track 1 — "Build a Checkpoint-Native +Developer Experience") submission by a solo developer. We are not modifying the +core CLI as an end in itself; we are building a new tool *on top* of it that +consumes Entire's own data — this repo's own Entire Checkpoints (the +`entire/checkpoints/v1` branch) and its Entire Graph. + +**What we're building — "Checkpoint Resume & Drift Assistant":** three CLI +subcommands over one shared core. + +- `entire resume` *(must-have)* — reconstructs a stopped/handed-off session's + context (stated intent, decisions, files touched) from the checkpoint + narrative, adds a Graph impact / blast-radius section for the files and + symbols the last session touched, and surfaces one Databricks aggregate + (e.g. requirement-coverage %). Renders to the terminal and to a single + self-contained HTML report. +- `entire drift` *(must-have)* — compares the original plan captured in the + first checkpoint against current code state using entity-level + `entire graph diff` (not raw text diff), and lists unfinished or dropped + requirements. +- `entire assess ` *(stretch goal, only if time remains)* — scopes + Graph impact analysis to one commit and cross-references it against the + nearest checkpoint's stated intent. + +**Shared core:** a **CheckpointReader** (parses `entire/checkpoints/v1` into +structured per-session records: id, timestamps, prompt/decision text, files +touched, stated intent), a **GraphClient** (thin wrapper over `entire graph +search` / `diff` / `impact` output), and a **DatabricksSync** layer that pushes +parsed records into a single Delta table queried back for trend/aggregate +analysis. + +**Read [`BUILDATHON_HANDOFF.md`](BUILDATHON_HANDOFF.md) in the repo root first** +for the complete picture: event rules and timeline, the required checkpoint +milestones and Graph-evidence obligations, the judging rubric, Databricks +constraints and the "meaningful use" bar, and every strategic decision and open +question from planning. + +**How to read the rest of this file:** everything below is the upstream Entire +CLI's own contributor and build documentation. Follow it only where it is +directly relevant to using `entire` and `entire graph` *as a tool consumer* — +command behavior and flags, checkpoint/branch and metadata layout, Graph usage, +settings semantics, session-strategy internals we need to parse. The parts about +modifying, testing, linting, and releasing the core CLI itself (`mise run +check`, the pre-commit checklist, E2E agent suites, Go style rules, CI +enforcement) do **not** govern this build unless we end up patching CLI +internals directly, in which case they apply to that change only. + ## Architecture - CLI built with github.com/spf13/cobra and github.com/charmbracelet/huh @@ -1599,3 +1648,10 @@ if err := form.Run(); err != nil { ... } - Always use the accessibility helpers for any `huh` forms/prompts - Test new interactive features with `ACCESSIBLE=1` to ensure they work - The accessible mode is documented in `--help` output + + +This repo has the entire-graph code graph installed. Before exploring code with +grep/find/whole-file reads, read .entire/graph-agent.md — resolution-first guidance +for using graph retrieval, focused source inspection, and verification. +@.entire/graph-agent.md + diff --git a/docs/demo/drift.html b/docs/demo/drift.html new file mode 100644 index 0000000000..81b0fb17dd --- /dev/null +++ b/docs/demo/drift.html @@ -0,0 +1,62 @@ +Checkpoint Lens - Drift

Checkpoint Lens — Drift Report

Plan 01M1TH4V7EFETXQ0QGNR2E871W → current 01M1TQ2SJR65VG107JGSEQ9MTA
33%
requirement coverage
18
requirements extracted
12
still open
257
entity changes

Plan vs implementation

RequirementVerdictEvidence
Edit CLAUDE.md: add a new top-level section titledPARTIALclaudeHookConfig cmd/entire/cli/agent/claudecode/hooks.go func claudeHookConfig(ctx contex
file's contributor/build instructions should only be followed if directlyPARTIALbuildTokensProfileReport cmd/entire/cli/tokens_profile.go func buildTokensProfileReport(ct
anything that changes how we should design CheckpointReaderPARTIALCheckpointReader tools/checkpoint_lens/reader.py class CheckpointReader:
Create a change-risk or release-readiness reportPARTIALcreateMirrors cmd/entire/cli/repo_mirror_create_wizard.go // than aborting the whole run.
Three CLI subcommands sharing one core (build in this priority order — 1 and 2 are must-haves, 3 is a stretch goal only if time remains)PARTIALrunCoreClient cmd/entire/cli/corecmd.go func runCoreClient(cmd *cobra.Command, newClient f
**CheckpointReader** — parses the `entire/checkpoints/v1` branch into structured session data: session id, timestamps, prompts/decisions text, files touched, stated intentIMPLEMENTEDCheckpointReader tools/checkpoint_lens/reader.py class CheckpointReader: """Reads checkpoi
**GraphClient** — thin wrapper shelling out to `entire graph` commands (`search`, `diff`, `impact`) and parsing their outputPARTIALGraphClient tools/checkpoint_lens/graph.py class GraphClient: def __init__(self, repo: str
**DatabricksSync** — pushes parsed checkpoint records into a Delta tablePARTIALDatabricksSync tools/checkpoint_lens/databricks.py class DatabricksSync: def __init__(self
Adds a Graph impact/blast-radius section: for files/symbols the last session touched, show what else depends on themIMPLEMENTED_graph_blast_radius tools/checkpoint_lens/cli.py "GRAPH IMPACT (BLAST RADIUS)",
Uses `entire graph diff` (entity-level) rather than raw text diff — a function can move without meaningfully changing; raw diff would false-flag it as drift, Graph won'tIMPLEMENTEDdiff tools/checkpoint_lens/graph.py def diff(self, base: str, head: str) -> GraphResult: "
Outputs a list of unfinished/dropped requirementsIMPLEMENTEDextract_requirements tools/checkpoint_lens/requirements.py def extract_requirements(texts:
Only build after 1 and 2 are solid and running on real dataPARTIALnormalizeRunnerID cmd/entire/cli/runner_prompt.go func normalizeRunnerID(id string) string
Cheap to add (~30–45 min) since it's composition of existing pieces, not new buildPARTIALgeminiModel e2e/agents/gemini.go func geminiModel() string { if m := os.Getenv("E2E_GEMINI
**Free Edition account** — create/open before the event (setup ≠ implementation, this is allowed prep)PARTIALencodeCreateServiceAccountRequest internal/coreapi/oas_request_encoders_gen.go func encode
A graph search or definition lookup — do early, right after `init-agents`IMPLEMENTEDsearch tools/checkpoint_lens/graph.py def search(self, query: str, profile: str = "full",
| 11:00–11:45 | Ship `entire resume` v1 and `entire drift` v1 — must run on real data from this repo, not mocked. |PARTIALrestoreResumeSessions cmd/entire/cli/resume.go func restoreResumeSessions(ctx context.Cont
[ ] Fallback screenshot/recording saved locallyPARTIALMutateSessionStateOnSaved cmd/entire/cli/strategy/session_state.go func MutateSessionState
Selected Entire track (Track 1)IMPLEMENTEDuninstallDeselectedAgentHooks cmd/entire/cli/setup.go func uninstallDeselectedAgentHooks(c

no matching code found - VERIFY before trusting; absence of evidence is not proof

Entity-level change since the plan

$ entire graph diff --repo . --base 34c1528d880985c9edb2267b0ca0178d916937a4 --head 79743a33261b8fbf471c091915c8010ae6354e83 --json

Entity-level, so a function that merely moved is not counted as drift.

ChangeFileDependents
body_changed document .gitignore.gitignore53
added section Checkpoint-LensBUILDATHON.md
added section One-sentence-summaryBUILDATHON.md
added section Problem-intended-user-and-why-it-mattersBUILDATHON.md
added section Selected-Entire-track-and-why-Entire-is-essentialBUILDATHON.md
added section Architecture-and-main-workflowBUILDATHON.md
added code_fence code_fence_1_textBUILDATHON.md
added code_fence code_fence_2_textBUILDATHON.md
added section Entire-Graph-findings-and-verificationBUILDATHON.md
added section Noon-Curveball:-what-changed-and-how-we-adaptedBUILDATHON.md
added section Checkpoint-links-and-what-each-checkpoint-provesBUILDATHON.md
added section Setup-run-and-test-instructionsBUILDATHON.md
added code_fence code_fence_3_bashBUILDATHON.md
added section Run-directly-no-install-needed-:BUILDATHON.md
added section Or-as-a-first-class-Entire-subcommand-put-the-repo-root-on-PATH-:BUILDATHON.md
added section PowerShell:-env:PATH-PWD-env:PATHBUILDATHON.md
added section bash:-export-PATH-PWD:-PATHBUILDATHON.md
added section Every-command-supports---json---no-graph-and---no-databricks.BUILDATHON.md
added code_fence code_fence_4_textBUILDATHON.md
added code_fence code_fence_5_bashBUILDATHON.md
added section 40-testsBUILDATHON.md
added code_fence code_fence_6_textBUILDATHON.md
added code_fence code_fence_7_jsonBUILDATHON.md
added code_fence code_fence_8_textBUILDATHON.md
added section Databricks-use-data-sources-and-limitationsBUILDATHON.md
added section Known-limitations-and-next-stepsBUILDATHON.md
added module entire-lensentire-lens
added module tools/checkpoint_lens/__init__.pytools/checkpoint_lens/__init__.py
added function _configure_stdiotools/checkpoint_lens/cli.py1
added function _emittools/checkpoint_lens/cli.py4
added function _loadtools/checkpoint_lens/cli.py4
added function _selecttools/checkpoint_lens/cli.py1
added function cmd_handofftools/checkpoint_lens/cli.py1
added function _aggregates_for_htmltools/checkpoint_lens/cli.py1
added function _write_htmltools/checkpoint_lens/cli.py2
added function _graph_blast_radiustools/checkpoint_lens/cli.py1
added function cmd_drifttools/checkpoint_lens/cli.py1
added function checktools/checkpoint_lens/cli.py462
added function _search_hitstools/checkpoint_lens/cli.py2
added function cmd_assesstools/checkpoint_lens/cli.py1
added function _resolve_revtools/checkpoint_lens/cli.py1
added function _nearest_checkpointtools/checkpoint_lens/cli.py1
added function _databricks_sectiontools/checkpoint_lens/cli.py3
added function cmd_synctools/checkpoint_lens/cli.py1
added function build_parsertools/checkpoint_lens/cli.py1
added function commontools/checkpoint_lens/cli.py119
added function maintools/checkpoint_lens/cli.py563
added function is_generated_pathtools/checkpoint_lens/databricks.py6
added class DatabricksConfigtools/checkpoint_lens/databricks.py9
added method DatabricksConfig.configuredtools/checkpoint_lens/databricks.py202
added method DatabricksConfig.fq_schematools/checkpoint_lens/databricks.py4
added method DatabricksConfig.tabletools/checkpoint_lens/databricks.py46
added method DatabricksConfig.resolvetools/checkpoint_lens/databricks.py405
added class SyncResulttools/checkpoint_lens/databricks.py2
added class AggregateResulttools/checkpoint_lens/databricks.py5
added class DatabricksSynctools/checkpoint_lens/databricks.py5
added method DatabricksSync.__init__tools/checkpoint_lens/databricks.py4
added method DatabricksSync.connector_availabletools/checkpoint_lens/databricks.py2
added method DatabricksSync.unavailable_reasontools/checkpoint_lens/databricks.py8
added method DatabricksSync._connecttools/checkpoint_lens/databricks.py3
MISSING means no evidence was found, not that the work is absent. Verify each open item against source before acting.
\ No newline at end of file diff --git a/docs/demo/handoff.html b/docs/demo/handoff.html new file mode 100644 index 0000000000..980a81ad61 --- /dev/null +++ b/docs/demo/handoff.html @@ -0,0 +1,111 @@ +Checkpoint Lens - Handoff

Checkpoint Lens — Handoff Brief

01M1TQ2SJR65VG107JGSEQ9MTA · 2026-09-06T06:39:22.5086715Z · branch RV/Entire_Trunk
7
files touched
19
decisions recovered
0%
agent-written
0
lines changed
29,527,987
tokens

Stated intent

Workspace URL: https://dbc-fbf9202b-29fc.cloud.databricks.com/browse/folders/workspace?o=7474654658489311 + +PAT: REDACTED +HTTP Path: /sql/1.0/warehouses/08d05243c52e3597
Provenance: first substantive user prompt (verbatim, not generated)

Linked commit

79743a33261b — feat(lens): make recovered decisions trustworthy; HTML reports; data-quality fixes

Decisions, risks and open questions

Recovered verbatim from checkpoint context — never paraphrased or generated.

OPEN
Previously the classifier scored any sentence containing a keyword, so the agent quoting its own task back ("open questions we still need to decide", "Commit everything with a substantive message") outscored genuine reasoning.
source: commit message · confidence: high
+
OPEN
[OPEN] open questions we still need to decide.
source: session transcript · confidence: medium
+
RISK
No tests yet. Biggest quality risk; rubric explicitly wants tests covering critical + curveball behavior.
source: session transcript · confidence: medium
+
REJECTED
Substring matching on bare nouns was wrong in a way that produced confident nonsense: "risk" matched the column name "file-churn/risk score", and "abandoned" matched a sentence *defining* what checkpoints preserve.
source: commit message · confidence: high
+
REJECTED
Markers are now regexes requiring the term to be predicated of the work ("we abandoned", "the risk is").
source: commit message · confidence: high
+
REJECTED
Three remaining defects I can see: sentences starting mid-way (hard-wrapped markdown split on newlines), instead of over-triggering REJECTED, and raw markup.
source: session transcript · confidence: medium
+
REJECTED
Clean full sentences, no false REJECTED.
source: session transcript · confidence: medium
+
REJECTED
Now RISK over-matches — it's catching the *word* "risk" in "file-churn/risk score", and "abandoned" inside a definition.
source: session transcript · confidence: medium
+
DECIDED
Threshold is deliberately high: dropping a real decision is worse than keeping a borderline one.
source: commit message · confidence: high
+
DECIDED
Over-correcting then lost most real content, because engineering prose states decisions passively ("chosen over an LLM call", "idempotent, so re-running never double-counts").
source: commit message · confidence: high
+
WHY
The coverage headline counts rows in the decisions table rather than SUM(decision_count) from sessions; the two disagreed because one is deduplicated, so the headline could not be reproduced by querying the data.
source: commit message · confidence: high
+
WHY
Includes an inline-SVG trend chart, hand-rolled rather than pulled from a charting CDN.
source: commit message · confidence: high
+
WHY
Anything uncomputable renders as an explicit "unavailable" panel, because a silently missing section reads as a clean bill of health.
source: commit message · confidence: high
+
WHY
Two checkpoints are labelled as too few to be a trend rather than drawn as one.
source: commit message · confidence: high
+
WHY
The Graph client had a silent bug: wrong flag spelling made entire graph commit exit 0 while printing an error to stdout — an empty blast radius that looked like "no dependents." Caught by verifying against the real payload instead of trusting it.
source: session transcript · confidence: medium
+
WHY
4m01s → 58s, and now checking 18 requirements instead of 8.
source: session transcript · confidence: medium
+
WHY
Four real bugs found by verifying Graph output instead of trusting it — three were silent (wrong flag → exit 0 → empty section that reads as "no impact").
source: session transcript · confidence: medium
+
WHY
Two real problems now: recall collapsed to 9, and "No tests yet" appears 6 times — because each checkpoint stores the *full* session transcript, so later checkpoints re-contain earlier decisions.
source: session transcript · confidence: medium
+
WHY
The commit messages are full of decisions phrased as declarative prose ("The aggregates are chosen so that...", "Sync is idempotent...
source: session transcript · confidence: medium

Graph impact (blast radius)

+GRAPH IMPACT (BLAST RADIUS)
+------------------------------------------------------------------------------
+  evidence command (rerun to verify):
+    $ entire graph commit 79743a33261b8fbf471c091915c8010ae6354e83 --repo . --max-seconds 120 --json
+
+  81 entity change(s) across 7 file(s):
+    - body_changed function cmd_handoff              tools/checkpoint_lens/cli.py  <- 1 dependent(s)
+    - added function _aggregates_for_html            tools/checkpoint_lens/cli.py  <- 1 dependent(s)
+    - added function _write_html                     tools/checkpoint_lens/cli.py  <- 2 dependent(s)
+    - body_changed function cmd_drift                tools/checkpoint_lens/cli.py  <- 1 dependent(s)
+    - body_changed function build_parser             tools/checkpoint_lens/cli.py  <- 1 dependent(s)
+    - body_changed function common                   tools/checkpoint_lens/cli.py  <- 119 dependent(s)
+    - added function is_generated_path               tools/checkpoint_lens/databricks.py  <- 6 dependent(s)
+    - body_changed class DatabricksSync              tools/checkpoint_lens/databricks.py  <- 5 dependent(s)
+    - body_changed method DatabricksSync.sync        tools/checkpoint_lens/databricks.py  <- 195 dependent(s)
+    - signature_changed method DatabricksSync._query tools/checkpoint_lens/databricks.py  <- 4 dependent(s)
+    - signature_changed method DatabricksSync.covera tools/checkpoint_lens/databricks.py  <- 33 dependent(s)
+    - added function esc                             tools/checkpoint_lens/html.py  <- 18 dependent(s)
+    - added function _stat                           tools/checkpoint_lens/html.py  <- 2 dependent(s)
+    - added function _decisions_block                tools/checkpoint_lens/html.py  <- 1 dependent(s)
+    - added function _trend_svg                      tools/checkpoint_lens/html.py  <- 1 dependent(s)
+    ... 66 more
+
+  HIGH-RISK (signature/removal with dependents):
+    ! signature_changed method DatabricksSync.coverage in tools/checkpoint_lens/databricks.py (33 dependents)
+    ! signature_changed function extract_decisions in tools/checkpoint_lens/reader.py (22 dependents)
+    ! signature_changed method DatabricksSync._query in tools/checkpoint_lens/databricks.py (4 dependents)
+    ! signature_changed method CheckpointReader._read_session in tools/checkpoint_lens/reader.py (2 dependents)

Unresolved context over time (Databricks)

101M1TH201M1TM001M1TM001M1TN001M1TN201M1TQ

Files touched

  • tools/checkpoint_lens/cli.py
  • tools/checkpoint_lens/databricks.py
  • tools/checkpoint_lens/html.py
  • tools/checkpoint_lens/models.py
  • tools/checkpoint_lens/reader.py
  • tools/checkpoint_lens/report.py
  • tools/checkpoint_lens/tests/test_extraction_quality.py

Checkpoint history

CheckpointWhenCommit
01M1TH4V7EFETXQ0QGNR2E871W2026-09-06T04:55:38checkpoint(milestone-1): initial understanding and intended architecture
01M1TM6JKPX4MV0DV829FQ77GA2026-09-06T05:49:01checkpoint(milestone-2): last stable state before the Noon Curveball
01M1TM6ZCD0CP72K2HW6S945KY2026-09-06T05:49:14chore(lens): drop committed __pycache__ and ignore bytecode
01M1TM9WCDTJJBYQNFEDCBJ04N2026-09-06T05:50:49fix(lens): correct the graph commit interface and parse entities properly
01M1TNJQEJTDEK7Y6MSGR5KBBQ2026-09-06T06:13:07feat(lens): add drift, assess and Databricks sync; 40 tests
01M1TNPW8FD7E6AV3JX3KJ5RKM2026-09-06T06:15:23docs: add BUILDATHON.md
01M1TNVVBDT9YVNKF7G7P38WBN2026-09-06T06:18:06fix(lens): recover intent when a checkpoint carries no prompt.txt
01M1TQ2SJR65VG107JGSEQ9MTA2026-09-06T06:39:22feat(lens): make recovered decisions trustworthy; HTML reports; data-quality fixes
Generated by entire lens from real Entire Checkpoints. Every figure is derived from committed checkpoint data; nothing is synthesised. Graph findings are evidence, not proof — rerun the printed commands to verify.
\ No newline at end of file diff --git a/entire-lens b/entire-lens new file mode 100644 index 0000000000..ae6a6379a9 --- /dev/null +++ b/entire-lens @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Entire external-command shim: `entire lens ...` resolves to `entire-lens` +# on PATH via the CLI's kubectl-style external-command lookup. +exec python -m tools.checkpoint_lens.cli "$@" diff --git a/entire-lens.cmd b/entire-lens.cmd new file mode 100644 index 0000000000..f36b9f281d --- /dev/null +++ b/entire-lens.cmd @@ -0,0 +1,4 @@ +@echo off +REM Entire external-command shim: `entire lens ...` resolves to `entire-lens` +REM on PATH via the CLI's kubectl-style external-command lookup. +python -m tools.checkpoint_lens.cli %* diff --git a/tools/checkpoint_lens/__init__.py b/tools/checkpoint_lens/__init__.py new file mode 100644 index 0000000000..0134823f96 --- /dev/null +++ b/tools/checkpoint_lens/__init__.py @@ -0,0 +1,8 @@ +"""Checkpoint Lens - a checkpoint-native developer experience for the Entire CLI. + +Reads real Entire Checkpoint context (intent, decisions, attribution, files +touched) and turns it into four developer workflows: handoff, drift, assess +and sync. +""" + +__version__ = "0.1.0" diff --git a/tools/checkpoint_lens/cli.py b/tools/checkpoint_lens/cli.py new file mode 100644 index 0000000000..c89dee2a10 --- /dev/null +++ b/tools/checkpoint_lens/cli.py @@ -0,0 +1,919 @@ +"""Checkpoint Lens CLI. + +Resolved by the Entire CLI's kubectl-style external-command lookup: a binary +named ``entire-lens`` on PATH is invoked as ``entire lens ``. + +Subcommands map one-to-one onto the Track 1 use cases: + + handoff hand work to another developer or agent, and resume after a break + drift review an implementation against its stated intent, and list + unfinished requirements + assess assess one change against the intent of its nearest checkpoint + sync push parsed checkpoint records to Databricks and read the + cross-session aggregates back +""" + +from __future__ import annotations + +import argparse +import json +import sys +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from typing import Any + +from . import __version__ +from .graph import GraphClient +from .databricks import DatabricksSync +from .entities import parse_entity_changes, risky, touched_paths +from .models import CheckpointRecord +from .reader import CheckpointReader +from . import completeness +from . import html as htmlreport +from . import report +from . import requirements + + +def _configure_stdio() -> None: + """Checkpoint text is arbitrary Unicode written by an agent, but a Windows + console defaults to cp1252. Without this, a single arrow or em-dash in a + recovered decision crashes the whole report. Degrade the character rather + than the command. + """ + for stream in (sys.stdout, sys.stderr): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + try: + reconfigure(encoding="utf-8", errors="replace") + except (ValueError, OSError): + pass + + +def _emit(lines: list[str]) -> None: + sys.stdout.write("\n".join(lines) + "\n") + + +def _load(args: argparse.Namespace) -> tuple[CheckpointReader, list[CheckpointRecord]]: + reader = CheckpointReader(args.repo) + records = reader.read_all() + if not records: + sys.stderr.write( + "No Entire Checkpoints found in %s.\n" + "Checkpoint Lens reads real checkpoint context; it has nothing to\n" + "work from until at least one checkpointed commit exists.\n" + "Run `entire status` to confirm checkpoints are enabled.\n" % args.repo + ) + raise SystemExit(2) + return reader, records + + + +def _databricks_reason(args: argparse.Namespace) -> str | None: + """The Databricks availability reason, or None when it was not consulted. + + None and "" are different answers and the banner treats them differently: + "" means we asked and it is up, None means nobody asked. + """ + if getattr(args, "no_databricks", False): + return None + return DatabricksSync(args.repo).unavailable_reason() + + +def _completeness( + args: argparse.Namespace, + rec: CheckpointRecord, + reader: CheckpointReader, + graph_ok: bool | None, + graph_detail: str = "", +) -> Any: + return completeness.assess( + rec, + warnings=reader.warnings, + graph_ok=graph_ok, + graph_detail=graph_detail, + databricks_reason=_databricks_reason(args), + ) + + +def _select(records: list[CheckpointRecord], wanted: str | None) -> CheckpointRecord: + if not wanted: + return records[-1] + for rec in records: + if rec.checkpoint_id == wanted or rec.checkpoint_id.startswith(wanted): + return rec + sys.stderr.write("No checkpoint matching %r. Known ids:\n" % wanted) + for rec in records: + sys.stderr.write(" " + rec.checkpoint_id + "\n") + raise SystemExit(2) + + +# -------------------------------------------------------------------------- +# handoff +# -------------------------------------------------------------------------- + +def cmd_handoff(args: argparse.Namespace) -> int: + reader, records = _load(args) + rec = _select(records, args.checkpoint) + + # Resolved before anything is rendered: the banner has to lead the report, + # so its inputs must be known before the first line is emitted. + graph = GraphClient(args.repo) + graph_ok = None if args.no_graph else graph.available() + comp = _completeness(args, rec, reader, graph_ok) + + if args.json: + payload: dict[str, Any] = { + "completeness": comp.to_dict(), + "checkpoint": rec.to_dict(), + "warnings": reader.warnings, + "history_length": len(records), + } + print(json.dumps(payload, indent=2, default=str)) + return 0 + + lines: list[str] = [] + lines.extend(report.render_header("CHECKPOINT LENS - HANDOFF BRIEF", args.repo)) + lines.append( + " This brief reconstructs a stopped session from checkpoint context:" + ) + lines.append( + " what was intended, what was decided, and what is still open." + ) + lines.extend(report.render_completeness(comp)) + lines.extend(report.render_checkpoint_summary(rec)) + lines.extend(report.render_intent(rec)) + lines.extend(report.render_sessions(rec)) + + lines.append(report.section("DECISIONS, RISKS AND OPEN QUESTIONS")) + lines.append(" Recovered verbatim - never paraphrased or generated.") + lines.append("") + lines.extend( + report.render_decisions( + rec.all_decisions(), + limit=args.limit, + transcript_available=any(s.transcript_available for s in rec.sessions), + ) + ) + + lines.extend(report.render_files(rec)) + + # Graph blast radius over the files this session actually touched. + graph_lines: list[str] = [] + if not args.no_graph: + graph_lines = _graph_blast_radius(args.repo, rec) + lines.extend(graph_lines) + + if len(records) > 1: + lines.append(report.section("SESSION HISTORY")) + for r in records: + lines.append( + " %s %s %s" + % ( + r.checkpoint_id, + (r.created_at or "")[:19] or "unknown-time", + (r.linked_subject or "(unlinked)")[:40], + ) + ) + + lines.extend(_databricks_section(args, headline_only=True)) + lines.extend(report.render_warnings(reader.warnings)) + lines.extend( + report.footer( + [ + "Intent provenance is stated above; verify any generated text " + "against the linked commit before acting on it.", + "Next: `entire lens drift` to see which stated requirements are " + "still unfinished.", + ] + ) + ) + _emit(lines) + + if args.html: + agg = _aggregates_for_html(args) + _write_html( + args.html, + htmlreport.render_handoff( + rec, records, reader.warnings, graph_lines, agg, comp + ), + ) + return 0 + + +def _aggregates_for_html(args: argparse.Namespace) -> dict[str, Any]: + """Trend rows for the HTML chart, or the reason there are none.""" + if getattr(args, "no_databricks", False): + return {"unavailable": "skipped with --no-databricks"} + sync = DatabricksSync(args.repo) + reason = sync.unavailable_reason() + if reason: + return {"unavailable": reason} + trend = sync.open_items_trend() + if not trend.ok: + return {"unavailable": trend.error[:200]} + return {"trend": trend.rows} + + +def _write_html(path: str, markup: str) -> None: + with open(path, "w", encoding="utf-8") as fh: + fh.write(markup) + sys.stdout.write("\nHTML report written to %s\n" % path) + + +def _graph_blast_radius(repo: str, rec: CheckpointRecord) -> list[str]: + """Ask the graph what else depends on what this session touched.""" + graph = GraphClient(repo) + if not graph.available(): + return report.render_graph_evidence( + "GRAPH IMPACT (BLAST RADIUS)", + "entire graph impact --repo %s --symbol " % repo, + [], + ok=False, + error="entire graph plugin not available on this machine", + ) + + target = rec.linked_commit or "" + if not target: + return report.render_graph_evidence( + "GRAPH IMPACT (BLAST RADIUS)", + "entire graph checkpoint --repo %s --id %s" % (repo, rec.checkpoint_id), + [], + ok=False, + error="checkpoint has no linked commit to analyze", + ) + + res = graph.commit_entities(target) + body: list[str] = [] + if res.ok: + changes = parse_entity_changes(res.data) + if changes: + paths = touched_paths(changes) + body.append( + " %d entity change(s) across %d file(s):" + % (len(changes), len(paths)) + ) + for c in changes[:15]: + suffix = "" + if c.dependents_count: + suffix = " <- %d dependent(s)" % c.dependents_count + body.append(" - %-46s %s%s" % (c.label[:46], c.path, suffix)) + if len(changes) > 15: + body.append(" ... %d more" % (len(changes) - 15)) + danger = risky(changes) + if danger: + body.append("") + body.append(" HIGH-RISK (signature/removal with dependents):") + for c in danger[:5]: + body.append( + " ! %s in %s (%d dependents)" + % (c.label, c.path, c.dependents_count) + ) + elif res.text.strip(): + for line in res.text.strip().splitlines()[:12]: + body.append(" " + line) + return report.render_graph_evidence( + "GRAPH IMPACT (BLAST RADIUS)", + res.command_str, + body, + ok=res.ok, + error=res.error, + ) + + +# -------------------------------------------------------------------------- +# drift +# -------------------------------------------------------------------------- + +def cmd_drift(args: argparse.Namespace) -> int: + """Review the implementation against the plan captured in the FIRST + checkpoint, and list what is still unfinished.""" + reader, records = _load(args) + baseline = records[0] + head = records[-1] + + reqs = requirements.extract_requirements( + [baseline.intent] + [p for s in baseline.sessions for p in s.prompts], + limit=args.limit, + ) + + graph = GraphClient(args.repo) + graph_ok = (not args.no_graph) and graph.available() + + def check(req: requirements.Requirement) -> requirements.DriftFinding: + finding = requirements.DriftFinding(requirement=req) + if not graph_ok: + finding.verdict = requirements.UNVERIFIED + finding.command = "entire graph search --repo %s --query ..." % args.repo + return finding + query = " ".join(requirements.keywords(req.text, limit=6)) or req.text[:80] + res = graph.search(query, profile=args.profile, limit=8) + finding.command = res.command_str + if not res.ok: + finding.verdict = requirements.UNVERIFIED + else: + hits = _search_hits(res.data, res.text) + finding.evidence = hits[:5] + finding.verdict, finding.score = requirements.classify(hits, req) + return finding + + # One graph search per requirement, run concurrently. Each search is an + # independent subprocess, so the wall-clock cost is the slowest query + # rather than their sum - 18 sequential `full`-profile searches took over + # four minutes, which is not a tool anyone would run twice. + if graph_ok and len(reqs) > 1: + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + findings = list(pool.map(check, reqs)) + else: + findings = [check(r) for r in reqs] + + # Entity-level diff between the plan's commit and now. Entity-level is the + # point: a function that merely moved is not drift. + diff_res = None + entity_changes: list[Any] = [] + if graph_ok and baseline.linked_commit and head.linked_commit: + diff_res = graph.diff(baseline.linked_commit, head.linked_commit) + if diff_res.ok: + entity_changes = parse_entity_changes(diff_res.data) + + # Drift compares two checkpoints, so its context is only as complete as + # the WEAKER of the two. Assessing the baseline alone would let a fully + # readable head hide a baseline whose plan we could not actually read - + # and the plan is the half the whole comparison rests on. + # None means "not consulted"; False means "asked and it could not answer". + # The banner phrases those differently, so the distinction has to survive + # the call rather than collapsing into a bare bool. + graph_state = None if args.no_graph else graph_ok + comp = _completeness( + args, baseline, reader, graph_state, "entire graph did not answer" + ) + head_comp = _completeness(args, head, reader, graph_state, "entire graph did not answer") + if len(head_comp.degraded) > len(comp.degraded): + comp = head_comp + + if args.json: + print( + json.dumps( + { + "completeness": comp.to_dict(), + "baseline_checkpoint": baseline.checkpoint_id, + "head_checkpoint": head.checkpoint_id, + "findings": [ + { + "requirement": f.requirement.text, + "origin": f.requirement.origin, + "verdict": f.verdict, + "score": round(f.score, 2), + "evidence": f.evidence, + "command": f.command, + } + for f in findings + ], + "entity_changes": [vars(c) for c in entity_changes], + "warnings": reader.warnings, + }, + indent=2, + default=str, + ) + ) + return 0 + + lines: list[str] = [] + lines.extend(report.render_header("CHECKPOINT LENS - DRIFT REPORT", args.repo)) + lines.append(" Plan (baseline) : %s" % baseline.checkpoint_id) + lines.append(" %s" % (baseline.linked_subject or "(unlinked)")) + lines.append(" Current (head) : %s" % head.checkpoint_id) + lines.append(" %s" % (head.linked_subject or "(unlinked)")) + + lines.extend(report.render_completeness(comp)) + lines.extend(report.render_drift_findings(findings)) + + lines.append(report.section("ENTITY-LEVEL CHANGE SINCE THE PLAN")) + if not graph_ok: + lines.append(" graph unavailable - entity-level comparison skipped.") + elif diff_res is None: + lines.append(" baseline or head checkpoint has no linked commit.") + elif not diff_res.ok: + lines.append(" graph diff failed: " + diff_res.error) + else: + lines.append(" evidence command (rerun to verify):") + lines.append(" $ " + diff_res.command_str) + lines.append("") + if entity_changes: + lines.append( + " %d entity change(s) across %d file(s). Entity-level, so a" + % (len(entity_changes), len(touched_paths(entity_changes))) + ) + lines.append(" function that only moved is not counted as drift.") + for c in entity_changes[:12]: + lines.append(" - %-44s %s" % (c.label[:44], c.path)) + if len(entity_changes) > 12: + lines.append(" ... %d more" % (len(entity_changes) - 12)) + else: + lines.append(" (graph reported no entity changes)") + + lines.extend(_databricks_section(args, headline_only=True)) + lines.extend(report.render_warnings(reader.warnings)) + lines.extend( + report.footer( + [ + "MISSING means no evidence was found, not that the work is " + "absent. Verify each open item against source before acting.", + ] + ) + ) + _emit(lines) + + open_items = [f for f in findings if f.is_open] + if args.fail_on_open and not comp.is_complete: + # A gate that goes green on context it could not fully read is worse + # than no gate: it converts "we do not know" into "we checked". + sys.stderr.write( + "\ndrift gate ran on PARTIAL context (%d of %d inputs missing or " + "redacted). A green result here means 'no drift found in what was " + "readable', not 'no drift'.\n" + % (len(comp.degraded), len(comp.inputs)) + ) + if args.fail_on_open and open_items: + sys.stderr.write( + "\ndrift gate FAILED: %d of %d stated requirement(s) have no " + "complete implementation evidence.\n" % (len(open_items), len(findings)) + ) + + if args.html: + _write_html( + args.html, + htmlreport.render_drift( + baseline, + head, + findings, + entity_changes, + diff_res.command_str if diff_res else "", + reader.warnings, + comp, + ), + ) + # Non-zero lets drift run as a release gate in CI: "did we build what the + # plan said we would?" fails the pipeline the same way a red test does. + if args.fail_on_open and open_items: + return 1 + return 0 + + +def _search_hits(data: Any, text: str) -> list[str]: + """Flatten graph search output into strings a requirement can be scored + against. + + ``entire graph search --format json`` returns ``results[]`` carrying + ``file_path`` plus line spans, and usually a ``snippet`` of the matched + source. Both matter: the path says *where* the implementation lives, the + snippet says *what* it is, and a requirement's keywords can legitimately + match either. + """ + out: list[str] = [] + if isinstance(data, dict): + results = data.get("results") + if isinstance(results, list): + for item in results: + if isinstance(item, str): + out.append(item) + continue + if not isinstance(item, dict): + continue + parts: list[str] = [] + for key in ("symbol", "symbol_name", "name", "container", "file_path", "path"): + val = item.get(key) + if isinstance(val, str) and val: + parts.append(val) + snippet = item.get("snippet") or item.get("text") or "" + if isinstance(snippet, str) and snippet: + parts.append(" ".join(snippet.split())[:300]) + if parts: + out.append(" ".join(parts)) + if out: + return out + if not out and text.strip(): + for line in text.strip().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + out.append(line) + return out + + +# -------------------------------------------------------------------------- +# assess +# -------------------------------------------------------------------------- + +def cmd_assess(args: argparse.Namespace) -> int: + """Assess one change against the stated intent of its nearest checkpoint.""" + reader, records = _load(args) + target = args.commitish + + by_commit = {r.linked_commit: r for r in records if r.linked_commit} + resolved = _resolve_rev(args.repo, target) + rec = by_commit.get(resolved) + nearest_note = "exact checkpoint for this commit" + if rec is None: + rec = _nearest_checkpoint(args.repo, records, resolved) + nearest_note = "nearest preceding checkpoint (this commit has none of its own)" + + graph = GraphClient(args.repo) + graph_ok = (not args.no_graph) and graph.available() + changes: list[Any] = [] + res = None + if graph_ok: + res = graph.commit_entities(resolved or target) + if res.ok: + changes = parse_entity_changes(res.data) + + changed_paths = set(touched_paths(changes)) or set( + reader.commit_files(resolved or target) + ) + planned = set(rec.files_touched) if rec else set() + unplanned = sorted(changed_paths - planned) if planned else [] + + comp = ( + _completeness(args, rec, reader, None if args.no_graph else graph_ok) + if rec is not None + else None + ) + + if args.json: + print( + json.dumps( + { + "completeness": comp.to_dict() if comp else { + "verdict": "PARTIAL", + "is_complete": False, + "inputs": [ + { + "name": "checkpoint context", + "status": "missing", + "detail": "this commit has no checkpoint to read", + } + ], + }, + "commit": resolved or target, + "checkpoint": rec.checkpoint_id if rec else None, + "checkpoint_relation": nearest_note, + "intent": rec.intent if rec else "", + "entity_changes": [vars(c) for c in changes], + "files_not_in_checkpoint": unplanned, + "warnings": reader.warnings, + }, + indent=2, + default=str, + ) + ) + return 0 + + lines: list[str] = [] + lines.extend(report.render_header("CHECKPOINT LENS - CHANGE ASSESSMENT", args.repo)) + lines.append(" commit: " + (resolved or target)) + if rec is None: + lines.append("") + lines.append(" No checkpoint context available for this commit.") + lines.append(" A change-risk view without intent is just a diff -") + lines.append(" run this against a checkpointed commit instead.") + _emit(lines) + return 1 + + lines.append(" checkpoint: %s (%s)" % (rec.checkpoint_id, nearest_note)) + lines.extend(report.render_completeness(comp)) + lines.extend(report.render_intent(rec)) + + lines.append(report.section("WHAT THIS CHANGE ACTUALLY DID (ENTITY-LEVEL)")) + if not graph_ok: + lines.append(" graph unavailable - entity analysis skipped.") + elif res is not None and not res.ok: + lines.append(" graph failed: " + res.error) + elif changes: + lines.append(" evidence command (rerun to verify):") + lines.append(" $ " + (res.command_str if res else "")) + lines.append("") + for c in changes[:15]: + suffix = " <- %d dependent(s)" % c.dependents_count if c.dependents_count else "" + lines.append(" - %-44s %s%s" % (c.label[:44], c.path, suffix)) + danger = risky(changes) + if danger: + lines.append("") + lines.append(" HIGH-RISK (signature change or removal with dependents):") + for c in danger[:5]: + lines.append( + " ! %s in %s (%d dependents)" % (c.label, c.path, c.dependents_count) + ) + else: + lines.append(" (no entity changes reported)") + + lines.append(report.section("INTENT CROSS-REFERENCE")) + if not planned: + lines.append(" The checkpoint records no file list to compare against.") + elif unplanned: + lines.append(" Files changed that the checkpoint's intent did NOT mention:") + for p in unplanned[:15]: + lines.append(" ? " + p) + lines.append("") + lines.append(" These are not necessarily wrong - they are the places") + lines.append(" where the change outgrew its stated plan, and the first") + lines.append(" thing a reviewer should ask about.") + else: + lines.append(" Every changed file appears in the checkpoint's file list.") + + if args.verify: + lines.extend(_verify_section(args, graph)) + + lines.extend(report.render_warnings(reader.warnings)) + _emit(lines) + return 0 + + +def _verify_section(args: argparse.Namespace, graph: GraphClient) -> list[str]: + """Run the project's tests and report an adjudicated verdict. + + An assessment that says what changed but not whether it still works is + half an answer. This is the half that survives a reviewer asking "and did + you run it?". + """ + lines = [report.section("VERIFICATION")] + if args.no_graph or not graph.available(): + lines.append(" graph unavailable - cannot adjudicate a test run.") + return lines + res, recorded = graph.verify(args.verify, args.verify_baseline) + lines.append(" evidence command (rerun to verify):") + lines.append(" $ " + res.command_str) + lines.append("") + if not res.ok: + lines.append(" verification FAILED to run: " + (res.error or "unknown")[:300]) + lines.append(" NOTE: a verifier that did not run is not a passing test.") + return lines + if recorded: + lines.append(" BASELINE RECORDED (%s)." % args.verify_baseline) + lines.append(" This run is a STATE, not a delta: it says which tests pass") + lines.append(" now, not which ones this change broke. Re-run after an edit") + lines.append(" to get a before/after verdict.") + lines.append("") + body = (res.text or "").strip() + if body: + for line in body.splitlines()[:20]: + lines.append(" " + line) + else: + lines.append(" (verifier returned no output)") + return lines + + +def _resolve_rev(repo: str, rev: str) -> str: + reader = CheckpointReader(repo) + try: + return reader._git("rev-parse", rev).strip() + except Exception: # noqa: BLE001 + return rev + + +def _nearest_checkpoint( + repo: str, records: list[CheckpointRecord], commit: str +) -> CheckpointRecord | None: + """The most recent checkpoint that is an ancestor of this commit.""" + reader = CheckpointReader(repo) + for rec in reversed(records): + if not rec.linked_commit: + continue + try: + reader._git("merge-base", "--is-ancestor", rec.linked_commit, commit) + return rec + except Exception: # noqa: BLE001 + continue + return records[-1] if records else None + + +# -------------------------------------------------------------------------- +# sync +# -------------------------------------------------------------------------- + +def _databricks_section(args: argparse.Namespace, headline_only: bool = False) -> list[str]: + """The cross-session aggregate. Unavailable is rendered as unavailable.""" + if getattr(args, "no_databricks", False): + return [] + sync = DatabricksSync(args.repo) + reason = sync.unavailable_reason() + lines = [report.section("CROSS-SESSION ANALYTICS (DATABRICKS)")] + if reason: + lines.append(" unavailable: " + reason) + lines.append(" Single-checkpoint views above are unaffected; only the") + lines.append(" cross-session trend and ranking are lost.") + return lines + cov = sync.coverage() + if not cov.ok: + lines.append(" query failed: " + cov.error[:200]) + return lines + row = cov.rows[0] if cov.rows else {} + lines.append( + " %s checkpoints | %s decisions captured | intent coverage %s%%" + % ( + row.get("checkpoints", "?"), + row.get("decisions_captured", "?"), + row.get("intent_coverage_pct", "?"), + ) + ) + lines.append( + " %s lines changed | avg %s%% agent-written" + % (row.get("lines_changed", "?"), row.get("avg_agent_pct", "?")) + ) + if headline_only: + return lines + + trend = sync.open_items_trend() + if trend.ok and trend.rows: + lines.append("") + lines.append(" Unresolved-context trend (blockers + open questions + risks):") + for r in trend.rows: + lines.append( + " %s %s unresolved=%s (blockers=%s open=%s risks=%s)" + % ( + str(r.get("checkpoint_id", ""))[:26], + str(r.get("created_at", ""))[:19], + r.get("unresolved_total", "?"), + r.get("blockers", "?"), + r.get("open_questions", "?"), + r.get("risks", "?"), + ) + ) + churn = sync.file_churn() + if churn.ok and churn.rows: + lines.append("") + lines.append(" File churn hotspots (touched across most checkpoints):") + for r in churn.rows[:8]: + lines.append( + " %-52s %s checkpoints" + % (str(r.get("file_path", ""))[:52], r.get("checkpoints_touching", "?")) + ) + return lines + + +def _egress_manifest(sync: DatabricksSync, records: list[CheckpointRecord]) -> list[str]: + """Print exactly what a sync would send, having sent nothing. + + Built from `DatabricksSync.build_rows` - the same function the real sync + uses - so this is the outgoing payload rather than a description of it. + Anyone can audit the egress boundary without credentials, without a + warehouse, and without trusting this file. + """ + from .databricks import EGRESS_COLUMNS + + built = sync.build_rows(records) + lines = [report.section("EGRESS MANIFEST (DRY RUN - NOTHING WAS SENT)")] + lines.append(" Only derived signals leave this machine: counts, kinds,") + lines.append(" enums, salted digests and identifiers. No prompt text and") + lines.append(" no transcript text, at any length.") + for table, rows in built.items(): + cols = EGRESS_COLUMNS[table] + lines.append("") + lines.append(" %s - %d row(s), %d column(s)" % (table, len(rows), len(cols))) + lines.append(" columns: " + ", ".join(c for c, _ in cols)) + if rows: + lines.append(" example row (real, from this repo):") + for (col, kind), value in zip(cols, rows[0]): + lines.append(" %-22s %-6s %r" % (col, kind, value)) + lines.append("") + lines.append(" Verified by assert_egress_safe: every value above matches") + lines.append(" its declared kind. A free-text value would have raised.") + return lines + + +def cmd_sync(args: argparse.Namespace) -> int: + reader, records = _load(args) + sync = DatabricksSync(args.repo) + + if args.dry_run: + # Deliberately ahead of the availability check: auditing what WOULD be + # sent must not require the ability to send it. + _emit(_egress_manifest(sync, records)) + return 0 + + reason = sync.unavailable_reason() + if reason: + sys.stderr.write("Databricks unavailable: " + reason + "\n") + return 2 + + if args.purge: + # Erasing data already sent is part of fixing a leak, not a separate + # nicety: stopping future writes leaves the old rows sitting there. + sys.stdout.write("Purging externally held checkpoint data:\n") + for statement in sync.purge_statements(): + sys.stdout.write(" " + statement + "\n") + res = sync.purge() + if not res.ok: + sys.stderr.write("purge failed: " + res.error + "\n") + return 1 + sys.stdout.write( + "Dropped %d table(s). Re-creating on the text-free schema...\n" + % len(res.detail) + ) + + if not args.query_only: + res = sync.sync(records) + if not res.ok: + sys.stderr.write("sync failed: " + res.error + "\n") + return 1 + sys.stdout.write( + "Synced %d session row(s), %d file row(s), %d decision row(s) " + "from %d checkpoint(s).\n" + % ( + res.sessions_written, + res.files_written, + res.decisions_written, + len(records), + ) + ) + + lines = _databricks_section(args, headline_only=False) + _emit(lines) + return 0 + + +# -------------------------------------------------------------------------- +# entry point +# -------------------------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="entire lens", + description=( + "Checkpoint Lens - turn real Entire Checkpoint context into " + "handoff, drift and change-assessment workflows." + ), + ) + p.add_argument("--version", action="version", version="checkpoint-lens " + __version__) + sub = p.add_subparsers(dest="command", required=True) + + def common(sp: argparse.ArgumentParser) -> None: + sp.add_argument( + "--html", metavar="PATH", + help="also write a self-contained HTML report to PATH", + ) + sp.add_argument("--repo", default=".", help="repository to read (default: .)") + sp.add_argument("--json", action="store_true", help="emit structured JSON") + sp.add_argument( + "--no-graph", action="store_true", help="skip entire graph calls" + ) + + h = sub.add_parser("handoff", help="reconstruct a session for handoff or resume") + common(h) + h.add_argument("--checkpoint", help="checkpoint id (default: most recent)") + h.add_argument("--limit", type=int, default=12, help="max decisions to show") + h.add_argument("--no-databricks", action="store_true", help="skip Databricks") + h.set_defaults(func=cmd_handoff) + + d = sub.add_parser("drift", help="review implementation against the original plan") + common(d) + d.add_argument("--limit", type=int, default=18, help="max requirements to check") + d.add_argument( + "--profile", default="fast", choices=["syntax-only", "fast", "full"], + help="graph parsing depth; 'full' is slower but resolves call graphs", + ) + d.add_argument("--jobs", type=int, default=6, help="concurrent graph searches") + d.add_argument( + "--fail-on-open", action="store_true", + help="exit non-zero if any stated requirement is unimplemented (CI gate)", + ) + d.add_argument("--no-databricks", action="store_true", help="skip Databricks") + d.set_defaults(func=cmd_drift) + + a = sub.add_parser("assess", help="assess one change against its checkpoint's intent") + common(a) + a.add_argument("commitish", help="commit to assess (sha, tag, HEAD~1, ...)") + a.add_argument("--no-databricks", action="store_true", help="skip Databricks") + a.add_argument( + "--verify", metavar="TEST_CMD", + help="run this test command and report an adjudicated pass/fail verdict", + ) + a.add_argument( + "--verify-baseline", metavar="PATH", default=".entire/lens-verify-baseline.json", + help="baseline file for adjudicated verification (recorded if absent)", + ) + a.set_defaults(func=cmd_assess) + + y = sub.add_parser("sync", help="push checkpoint records to Databricks and read aggregates back") + common(y) + y.add_argument("--query-only", action="store_true", help="do not write; only read aggregates") + y.add_argument( + "--dry-run", action="store_true", + help="print the exact rows a sync would send and exit; connects to nothing", + ) + y.add_argument( + "--purge", action="store_true", + help="drop the remote tables before syncing (erases previously sent data)", + ) + y.add_argument("--no-databricks", action="store_true", help=argparse.SUPPRESS) + y.set_defaults(func=cmd_sync) + + return p + + +def main(argv: list[str] | None = None) -> int: + _configure_stdio() + parser = build_parser() + args = parser.parse_args(argv) + return int(args.func(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/checkpoint_lens/completeness.py b/tools/checkpoint_lens/completeness.py new file mode 100644 index 0000000000..8840e4f318 --- /dev/null +++ b/tools/checkpoint_lens/completeness.py @@ -0,0 +1,201 @@ +"""One authoritative answer to "how much of this is actually here?". + +WHY THIS MODULE EXISTS +---------------------- +Checkpoint Lens already reported degraded inputs - a warnings block, a "graph +unavailable" line, a "Databricks unavailable" line, an intent-provenance label. +Five separate signals, each true, each local to its own section, none of them +at the top, and none of them covering a missing transcript, an absent +metadata.json or content that Entire had already redacted. + +A reader who skims a report and stops before the warnings block therefore read +a partial reconstruction as a complete one. That is the failure this module +removes: ONE banner, computed once, rendered first, in every output mode +(terminal, HTML and --json). The per-section notes stay as the detail. + +THE RULE THIS ENCODES +--------------------- +Absence of evidence is not evidence of absence. A section that says "no +blockers found" means one thing when the transcript was read and another when +there was no transcript to read, and the product must never let those two look +alike. Every input is therefore reported positively - readable or not - rather +than only mentioned when it fails. + +REDACTION IS A THIRD STATE +-------------------------- +An input can be present, readable, and still incomplete, because Entire's +redaction pipeline removed content before the checkpoint was ever committed. +That is neither "available" nor "missing": the checkpoint is intact and some of +what it described is gone. It is reported as REDACTED and counts as partial. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from .models import CheckpointRecord, INTENT_UNAVAILABLE + +# Status values. REDACTED is deliberately distinct from MISSING: conflating +# them would either overstate what we hold or understate what was removed. +AVAILABLE = "available" +MISSING = "missing" +REDACTED = "redacted" + + +@dataclass +class Input: + """One input the report is built from, and whether we actually had it.""" + + name: str + status: str + detail: str = "" + + @property + def ok(self) -> bool: + return self.status == AVAILABLE + + +@dataclass +class ContextCompleteness: + inputs: list[Input] = field(default_factory=list) + + @property + def degraded(self) -> list[Input]: + return [i for i in self.inputs if not i.ok] + + @property + def is_complete(self) -> bool: + return not self.degraded + + @property + def verdict(self) -> str: + return "COMPLETE" if self.is_complete else "PARTIAL" + + def to_dict(self) -> dict[str, Any]: + """The same verdict --json consumers get. An agent reading this must be + able to branch on `is_complete` without parsing prose.""" + return { + "verdict": self.verdict, + "is_complete": self.is_complete, + "inputs_total": len(self.inputs), + "inputs_available": len(self.inputs) - len(self.degraded), + "inputs": [ + {"name": i.name, "status": i.status, "detail": i.detail} + for i in self.inputs + ], + } + + +def assess( + rec: CheckpointRecord, + warnings: list[str] | None = None, + graph_ok: bool | None = None, + graph_detail: str = "", + databricks_reason: str | None = None, +) -> ContextCompleteness: + """Assess one checkpoint's context. + + `graph_ok` and `databricks_reason` are ``None`` when the caller did not + consult that subsystem at all (``--no-graph`` / ``--no-databricks``). That + is still not complete context - the user chose to run without it, and the + banner says so rather than quietly counting it as fine. + """ + inputs: list[Input] = [] + sessions = rec.sessions or [] + + # --- session metadata -------------------------------------------------- + if not sessions: + inputs.append(Input("session metadata", MISSING, "this checkpoint records no sessions")) + elif all(s.metadata_available for s in sessions): + inputs.append(Input("session metadata", AVAILABLE)) + else: + n = sum(1 for s in sessions if not s.metadata_available) + inputs.append( + Input( + "session metadata", + MISSING, + "%d of %d session(s) have no readable metadata.json; agent, " + "turn and token figures for those are unknown, not zero" + % (n, len(sessions)), + ) + ) + + # --- stated intent ----------------------------------------------------- + if not rec.intent or rec.intent_source == INTENT_UNAVAILABLE: + inputs.append( + Input("stated intent", MISSING, "no summary, prompt.txt or transcript to recover it from") + ) + else: + inputs.append(Input("stated intent", AVAILABLE, "source: " + rec.intent_source)) + + # --- transcript -------------------------------------------------------- + if any(s.transcript_available for s in sessions): + inputs.append(Input("session transcript", AVAILABLE)) + else: + inputs.append( + Input( + "session transcript", + MISSING, + "decisions, risks and open questions could not be recovered at all", + ) + ) + + # --- redaction --------------------------------------------------------- + if any(s.redacted_content for s in sessions): + inputs.append( + Input( + "recovered text", + REDACTED, + "Entire's redaction pipeline removed content before this " + "checkpoint was committed; what it removed is not recoverable here", + ) + ) + else: + inputs.append(Input("recovered text", AVAILABLE, "no redaction markers found")) + + # --- files touched ----------------------------------------------------- + if rec.files_touched: + inputs.append(Input("files touched", AVAILABLE)) + else: + inputs.append(Input("files touched", MISSING, "no file list on the checkpoint or its commit")) + + # --- graph ------------------------------------------------------------- + if graph_ok is None: + inputs.append(Input("graph impact", MISSING, "not consulted (--no-graph)")) + elif graph_ok: + inputs.append(Input("graph impact", AVAILABLE)) + else: + inputs.append( + Input( + "graph impact", + MISSING, + (graph_detail or "entire graph did not answer") + + "; unavailable is not the same as 'no impact'", + ) + ) + + # --- cross-session analytics ------------------------------------------- + if databricks_reason is None: + inputs.append( + Input("cross-session analytics", MISSING, "not consulted (--no-databricks)") + ) + elif databricks_reason: + inputs.append(Input("cross-session analytics", MISSING, databricks_reason)) + else: + inputs.append(Input("cross-session analytics", AVAILABLE)) + + # --- reader warnings --------------------------------------------------- + warnings = warnings or [] + if warnings: + inputs.append( + Input( + "checkpoint read", + MISSING, + "%d warning(s) during the read - see the WARNINGS section" % len(warnings), + ) + ) + else: + inputs.append(Input("checkpoint read", AVAILABLE)) + + return ContextCompleteness(inputs=inputs) diff --git a/tools/checkpoint_lens/databricks.py b/tools/checkpoint_lens/databricks.py new file mode 100644 index 0000000000..0ae70157ff --- /dev/null +++ b/tools/checkpoint_lens/databricks.py @@ -0,0 +1,779 @@ +"""DatabricksSync - cross-session checkpoint analytics on Delta tables. + +WHY THIS IS LOAD-BEARING, NOT STORAGE +------------------------------------- +`handoff` and `assess` answer questions about ONE checkpoint, and they answer +them entirely from local git. Databricks exists here to answer the questions +that only have meaning across the whole history of a project: + + * Is unresolved context accumulating or being discharged over time? + (open_items_trend - a blocker recorded in checkpoint 1 and never answered + by checkpoint 6 is invisible to any single-checkpoint view.) + + * Which files are risk hotspots - repeatedly touched across many separate + sessions AND carrying many dependents? + (file_churn_risk - churn is a property of the sequence, not of one commit.) + +Delete this module and the product still runs, but every trend and ranking +degrades to a single point in time. That is the intended failure mode, and the +CLI states it explicitly rather than silently showing one checkpoint's numbers +as though they were a trend. + +CREDENTIALS +----------- +Never committed. Resolved in this order: + 1. environment: DATABRICKS_SERVER_HOSTNAME / DATABRICKS_HTTP_PATH / + DATABRICKS_TOKEN + 2. a gitignored .databricks.local.json in the repo root + +EGRESS CONTRACT +--------------- +This module is the ONLY code in Checkpoint Lens that opens a socket +(`_connect`, called by `sync` for writes and `_query` for reads). Everything it +sends is bound by one rule: + + Only DERIVED, NON-REVERSIBLE SIGNALS leave this machine: counts, kinds, + enums, salted digests, scores, and checkpoint/commit identifiers. No raw + prompt text and no transcript text is sent, at any length. + +Truncation is NOT part of that rule and never was a safeguard. A prompt cut at +800 characters is still 800 characters of the user's own words; it is a size +cap wearing a privacy label. The previous schema shipped `sessions.intent` and +`decisions.text` verbatim-then-truncated, which put raw prompts, transcript +prose and local filesystem paths into an external warehouse. Both columns are +gone. `derive_signals` is the single chokepoint that replaced them, and +`assert_egress_safe` fails the write if anything text-shaped ever reaches a row +again. + +What still leaves, deliberately: checkpoint metadata that is not prompt or +transcript content - `checkpoint_id`, `linked_commit`, `branch`, `session_id`, +`agent`, `model`, `created_at`, `file_path`, token counts, attribution +percentages. `file_churn` is unbuildable without `file_path`, and none of these +carry authored prose. That is a scope decision, stated so it can be revisited. + +The full verbatim text is never lost: it stays in the local checkpoints, and +`handoff` / `drift` / `assess` keep rendering it from git. Only the aggregate +layer is de-identified. + +DATA PROVENANCE +--------------- +Every row is derived from this repository's own Entire Checkpoints, created by +the team during the build. No third-party, customer or personal data is +uploaded. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import dataclass, field +from typing import Any, Iterable + +from .models import CheckpointRecord, has_redaction_marker +from .reader import attribute_to_first_appearance + +CONFIG_FILENAME = ".databricks.local.json" + +DEFAULT_CATALOG = "workspace" +DEFAULT_SCHEMA = "checkpoint_lens" + +# Build artefacts, vendored code and lockfiles are not authored work. They +# churn constantly and would otherwise dominate a hotspot ranking that is +# supposed to point a reviewer at risky *source*. Our own history contains a +# committed-then-deleted __pycache__, which sat at the top of the ranking until +# this filter existed. +GENERATED_PATH = re.compile( + r"(^|/)(__pycache__|node_modules|\.venv|venv|dist|build|vendor|target|" + r"\.mypy_cache|\.pytest_cache|coverage)(/|$)" + r"|\.(pyc|pyo|class|o|so|dll|exe|lock|min\.js|map)$" + r"|(^|/)(package-lock\.json|yarn\.lock|go\.sum|poetry\.lock)$", + re.IGNORECASE, +) + + +NEWLINE = re.compile("[\r\n]") + + +def is_generated_path(path: str) -> bool: + """True for build output, vendored code and lockfiles.""" + return bool(GENERATED_PATH.search(path.replace("\\", "/"))) + + +# -------------------------------------------------------------------------- +# The egress boundary +# -------------------------------------------------------------------------- +# +# Everything below exists to answer one question mechanically instead of by +# review: "can raw prompt or transcript text reach the wire?" + +DIGEST_DOMAIN = "entire-lens/v1/checkpoint-signal" +DIGEST_CHARS = 16 + +# What a column is allowed to hold. This table is the schema of record for the +# boundary: `assert_egress_safe` validates every outgoing row against it, so a +# column added later cannot quietly reintroduce free text the way `intent` and +# `text` did. +KIND_COUNT = "count" # int +KIND_SCORE = "score" # float +KIND_FLAG = "flag" # bool +KIND_DIGEST = "digest" # salted sha256 prefix, or empty +KIND_ENUM = "enum" # closed vocabulary; spaces allowed ("Claude Code") +KIND_IDENT = "ident" # id / sha / branch: no whitespace at all +KIND_PATH = "path" # repo-relative path; may contain spaces +KIND_TIME = "time" # ISO-8601 timestamp + +MAX_ENUM_CHARS = 40 +MAX_IDENT_CHARS = 128 +MAX_PATH_CHARS = 512 +MAX_TIME_CHARS = 40 + +HEX16 = re.compile("^[0-9a-f]{16}$") + +EGRESS_COLUMNS: dict[str, list[tuple[str, str]]] = { + "checkpoint_sessions": [ + ("checkpoint_id", KIND_IDENT), + ("session_index", KIND_COUNT), + ("session_id", KIND_IDENT), + ("created_at", KIND_TIME), + ("branch", KIND_IDENT), + ("agent", KIND_ENUM), + ("model", KIND_ENUM), + ("turn_count", KIND_COUNT), + ("save_step_count", KIND_COUNT), + ("prompt_count", KIND_COUNT), + ("intent_source", KIND_ENUM), + ("intent_len", KIND_COUNT), + ("intent_word_count", KIND_COUNT), + ("intent_digest", KIND_DIGEST), + ("intent_redacted", KIND_FLAG), + ("files_touched_count", KIND_COUNT), + ("decision_count", KIND_COUNT), + ("input_tokens", KIND_COUNT), + ("output_tokens", KIND_COUNT), + ("cache_read_tokens", KIND_COUNT), + ("cache_creation_tokens", KIND_COUNT), + ("api_call_count", KIND_COUNT), + ("total_tokens", KIND_COUNT), + ("agent_percentage", KIND_SCORE), + ("total_lines_changed", KIND_COUNT), + ("linked_commit", KIND_IDENT), + ("repo", KIND_IDENT), + ], + "checkpoint_files": [ + ("checkpoint_id", KIND_IDENT), + ("created_at", KIND_TIME), + ("file_path", KIND_PATH), + ("repo", KIND_IDENT), + ], + "checkpoint_decisions": [ + ("checkpoint_id", KIND_IDENT), + ("created_at", KIND_TIME), + ("kind", KIND_ENUM), + ("source", KIND_ENUM), + ("confidence", KIND_ENUM), + ("text_len", KIND_COUNT), + ("text_word_count", KIND_COUNT), + ("text_digest", KIND_DIGEST), + ("text_redacted", KIND_FLAG), + ("repo", KIND_IDENT), + ], +} + + +class EgressViolation(RuntimeError): + """A row carried a value the egress contract does not permit. + + Raised before any statement is executed. Failing the sync is the correct + outcome: an under-redacted upload cannot be un-sent, and a sync that did + not happen costs a trend chart. + """ + + +def _check_value(table: str, column: str, kind: str, value: Any) -> None: + if value is None: + return + if kind == KIND_FLAG: + if not isinstance(value, bool): + raise EgressViolation( + "%s.%s must be a bool, got %r" % (table, column, type(value)) + ) + return + if kind == KIND_COUNT: + if isinstance(value, bool) or not isinstance(value, int): + raise EgressViolation( + "%s.%s must be an int, got %r" % (table, column, type(value)) + ) + return + if kind == KIND_SCORE: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise EgressViolation( + "%s.%s must be numeric, got %r" % (table, column, type(value)) + ) + return + + if not isinstance(value, str): + raise EgressViolation( + "%s.%s must be a string, got %r" % (table, column, type(value)) + ) + + # No permitted column may span lines. Prose does; an id, a path and an + # enum do not. This is the cheapest tripwire for "someone put a transcript + # in here". + if NEWLINE.search(value): + raise EgressViolation( + "%s.%s contains a newline - free text is not permitted" % (table, column) + ) + + if kind == KIND_DIGEST: + if value and not HEX16.match(value): + raise EgressViolation( + "%s.%s is not a %d-char digest" % (table, column, DIGEST_CHARS) + ) + return + + cap = { + KIND_ENUM: MAX_ENUM_CHARS, + KIND_IDENT: MAX_IDENT_CHARS, + KIND_PATH: MAX_PATH_CHARS, + KIND_TIME: MAX_TIME_CHARS, + }[kind] + if len(value) > cap: + raise EgressViolation( + "%s.%s is %d chars, over the %d-char cap for %s - free text is not permitted" + % (table, column, len(value), cap, kind) + ) + if kind == KIND_IDENT and any(c.isspace() for c in value): + raise EgressViolation( + "%s.%s contains whitespace and is not an identifier" % (table, column) + ) + + +def assert_egress_safe(table: str, rows: list[list[Any]]) -> None: + """Validate every outgoing row against EGRESS_COLUMNS, or refuse to send. + + Called immediately before the INSERT rather than at row-construction time, + so a future caller that assembles rows some other way is still covered. + """ + spec = EGRESS_COLUMNS.get(table) + if spec is None: + raise EgressViolation("no egress column spec for table %r" % table) + for row in rows: + if len(row) != len(spec): + raise EgressViolation( + "%s row has %d values, spec has %d columns" % (table, len(row), len(spec)) + ) + for (column, kind), value in zip(spec, row): + _check_value(table, column, kind, value) + + +def derive_signals(text: str, repo_key: str) -> dict[str, Any]: + """Turn free text into the only things allowed to describe it externally. + + Returns length, word count, a salted digest, and whether Entire's redaction + pipeline had already removed content from it. The text itself is discarded + here and is never returned. + + On the digest, stated honestly: it is a stable key for dedupe and joins, + NOT an anonymisation primitive. A short, low-entropy string can be + recovered from any hash by guessing it. Salting with the repo key gives + domain separation so digests cannot be correlated across repositories, and + the plaintext never leaves this machine either way - that, not the hash, is + what makes this safe. + """ + text = text or "" + normalized = " ".join(text.split()) + digest = "" + if normalized: + h = hashlib.sha256() + h.update(DIGEST_DOMAIN.encode("utf-8")) + h.update(b"\x00") + h.update((repo_key or "").encode("utf-8")) + h.update(b"\x00") + h.update(normalized.encode("utf-8")) + digest = h.hexdigest()[:DIGEST_CHARS] + return { + "len": len(text), + "word_count": len(normalized.split()) if normalized else 0, + "digest": digest, + "redacted": has_redaction_marker(text), + } + + +@dataclass +class DatabricksConfig: + server_hostname: str = "" + http_path: str = "" + access_token: str = "" + catalog: str = DEFAULT_CATALOG + schema: str = DEFAULT_SCHEMA + + @property + def configured(self) -> bool: + return bool(self.server_hostname and self.http_path and self.access_token) + + @property + def fq_schema(self) -> str: + return "%s.%s" % (self.catalog, self.schema) + + def table(self, name: str) -> str: + return "%s.%s" % (self.fq_schema, name) + + @classmethod + def resolve(cls, repo: str = ".") -> "DatabricksConfig": + cfg = cls() + path = os.path.join(repo, CONFIG_FILENAME) + if os.path.isfile(path): + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + cfg.server_hostname = str(data.get("server_hostname", "") or "") + cfg.http_path = str(data.get("http_path", "") or "") + cfg.access_token = str(data.get("access_token", "") or "") + cfg.catalog = str(data.get("catalog", "") or DEFAULT_CATALOG) + cfg.schema = str(data.get("schema", "") or DEFAULT_SCHEMA) + except (OSError, json.JSONDecodeError, ValueError): + pass + # Environment wins: it is the path CI and other machines use. + cfg.server_hostname = os.environ.get("DATABRICKS_SERVER_HOSTNAME", cfg.server_hostname) + cfg.http_path = os.environ.get("DATABRICKS_HTTP_PATH", cfg.http_path) + cfg.access_token = os.environ.get("DATABRICKS_TOKEN", cfg.access_token) + cfg.catalog = os.environ.get("DATABRICKS_CATALOG", cfg.catalog) + cfg.schema = os.environ.get("DATABRICKS_SCHEMA", cfg.schema) + return cfg + + +@dataclass +class SyncResult: + ok: bool = False + sessions_written: int = 0 + files_written: int = 0 + decisions_written: int = 0 + error: str = "" + detail: list[str] = field(default_factory=list) + + +@dataclass +class AggregateResult: + """An aggregate answered by Databricks. `ok=False` means the number is + genuinely unknown - callers must render that as unavailable, never as + zero.""" + + name: str + ok: bool + rows: list[dict[str, Any]] = field(default_factory=list) + error: str = "" + sql: str = "" + + +DDL_SESSIONS = """ +CREATE TABLE IF NOT EXISTS {t} ( + checkpoint_id STRING, + session_index INT, + session_id STRING, + created_at STRING, + branch STRING, + agent STRING, + model STRING, + turn_count INT, + save_step_count INT, + prompt_count INT, + intent_source STRING, + intent_len INT, + intent_word_count INT, + intent_digest STRING, + intent_redacted BOOLEAN, + files_touched_count INT, + decision_count INT, + input_tokens BIGINT, + output_tokens BIGINT, + cache_read_tokens BIGINT, + cache_creation_tokens BIGINT, + api_call_count INT, + total_tokens BIGINT, + agent_percentage DOUBLE, + total_lines_changed INT, + linked_commit STRING, + repo STRING +) USING DELTA +""" + +DDL_FILES = """ +CREATE TABLE IF NOT EXISTS {t} ( + checkpoint_id STRING, + created_at STRING, + file_path STRING, + repo STRING +) USING DELTA +""" + +DDL_DECISIONS = """ +CREATE TABLE IF NOT EXISTS {t} ( + checkpoint_id STRING, + created_at STRING, + kind STRING, + source STRING, + confidence STRING, + text_len INT, + text_word_count INT, + text_digest STRING, + text_redacted BOOLEAN, + repo STRING +) USING DELTA +""" + +# Unresolved-context debt over time. A single checkpoint cannot answer this. +SQL_OPEN_ITEMS_TREND = """ +SELECT + d.checkpoint_id, + MIN(d.created_at) AS created_at, + SUM(CASE WHEN d.kind = 'blocker' THEN 1 ELSE 0 END) AS blockers, + SUM(CASE WHEN d.kind = 'open_question' THEN 1 ELSE 0 END) AS open_questions, + SUM(CASE WHEN d.kind = 'risk' THEN 1 ELSE 0 END) AS risks, + SUM(CASE WHEN d.kind IN ('blocker','open_question','risk') + THEN 1 ELSE 0 END) AS unresolved_total +FROM {decisions} d +WHERE d.repo = ? +GROUP BY d.checkpoint_id +ORDER BY created_at +""" + +# Churn hotspots: a property of the sequence of checkpoints, not of any one. +SQL_FILE_CHURN = """ +SELECT + f.file_path, + COUNT(DISTINCT f.checkpoint_id) AS checkpoints_touching, + MIN(f.created_at) AS first_seen, + MAX(f.created_at) AS last_seen +FROM {files} f +WHERE f.repo = ? +GROUP BY f.file_path +HAVING COUNT(DISTINCT f.checkpoint_id) >= 1 +ORDER BY checkpoints_touching DESC, f.file_path +LIMIT 15 +""" + +# Headline coverage numbers surfaced by handoff and drift. +# +# Note what this reads: intent_source, never the intent itself. That was +# already true before the intent column was removed, which is why removing it +# cost no aggregate anything. +# +# decisions_captured counts rows in the decisions table, NOT SUM(decision_count) +# from the sessions table. The two disagree on purpose: decision_count is the +# raw per-session count, while the decisions table is deduplicated to first +# appearance. Reporting the raw sum next to a deduplicated table showed a +# headline number no query against the data could reproduce. +SQL_COVERAGE = """ +SELECT + (SELECT COUNT(DISTINCT checkpoint_id) FROM {sessions} WHERE repo = ?) AS checkpoints, + (SELECT COUNT(*) FROM {decisions} WHERE repo = ?) AS decisions_captured, + (SELECT ROUND(AVG(agent_percentage), 1) FROM {sessions} WHERE repo = ?) AS avg_agent_pct, + (SELECT SUM(total_lines_changed) FROM {sessions} WHERE repo = ?) AS lines_changed, + (SELECT SUM(total_tokens) FROM {sessions} WHERE repo = ?) AS tokens, + (SELECT ROUND( + 100.0 * SUM(CASE WHEN intent_source <> 'unavailable' THEN 1 ELSE 0 END) + / NULLIF(COUNT(*), 0), 1) + FROM {sessions} WHERE repo = ?) AS intent_coverage_pct +""" + + +class DatabricksSync: + def __init__(self, repo: str = ".", config: DatabricksConfig | None = None) -> None: + self.repo = repo + self.config = config or DatabricksConfig.resolve(repo) + self._repo_key = os.path.basename(os.path.abspath(repo)) or "repo" + + # ---------------- availability ---------------- + + @staticmethod + def connector_available() -> bool: + try: + import databricks.sql # noqa: F401 + except ImportError: + return False + return True + + def unavailable_reason(self) -> str: + if not self.connector_available(): + return ( + "databricks-sql-connector is not installed " + "(pip install databricks-sql-connector)" + ) + if not self.config.configured: + return ( + "no Databricks credentials found - set DATABRICKS_SERVER_HOSTNAME / " + "DATABRICKS_HTTP_PATH / DATABRICKS_TOKEN, or create a gitignored " + + CONFIG_FILENAME + ) + return "" + + def _connect(self): + import databricks.sql + + return databricks.sql.connect( + server_hostname=self.config.server_hostname, + http_path=self.config.http_path, + access_token=self.config.access_token, + ) + + # ---------------- schema ---------------- + + def ensure_schema(self, cursor) -> None: + cursor.execute("CREATE SCHEMA IF NOT EXISTS " + self.config.fq_schema) + cursor.execute(DDL_SESSIONS.format(t=self.config.table("checkpoint_sessions"))) + cursor.execute(DDL_FILES.format(t=self.config.table("checkpoint_files"))) + cursor.execute(DDL_DECISIONS.format(t=self.config.table("checkpoint_decisions"))) + + # ---------------- purge ---------------- + + def purge_statements(self) -> list[str]: + """The exact statements `purge` will execute, for review before it runs. + + DROP, not DELETE, and the distinction is the whole point. The old + schema had `sessions.intent` and `decisions.text` columns holding raw + prompt and transcript text; deleting rows leaves those columns in + place, and Delta keeps the deleted rows reachable through time travel + until a VACUUM past the retention window. Dropping the table discards + the schema and its history together, and the tables are recreated + empty on the new, text-free schema by the sync that follows. + + Consequence worth stating plainly: DROP is not scoped by `repo`, so it + removes every repo's rows from this schema, not just ours. The column + removal is a schema change and cannot be scoped to one repo anyway. + Re-run `sync` in any other repo sharing this workspace afterwards. + """ + return [ + "DROP TABLE IF EXISTS " + self.config.table(t) + for t in ("checkpoint_sessions", "checkpoint_files", "checkpoint_decisions") + ] + + def purge(self) -> SyncResult: + """Drop the tables that held raw text. Recreation is left to `sync`.""" + reason = self.unavailable_reason() + if reason: + return SyncResult(ok=False, error=reason) + result = SyncResult() + try: + with self._connect() as conn: + with conn.cursor() as cur: + for statement in self.purge_statements(): + cur.execute(statement) + result.detail.append(statement) + result.ok = True + except Exception as exc: # noqa: BLE001 + result.ok = False + result.error = "%s: %s" % (type(exc).__name__, exc) + return result + + # ---------------- write ---------------- + + def build_rows(self, records: Iterable[CheckpointRecord]) -> dict[str, list[list[Any]]]: + """Every value that would leave this machine, and nothing else. + + Split out of `sync` so that `--dry-run` shows the ACTUAL outgoing rows + rather than a description of them. A manifest assembled by separate + code would be a second implementation to keep honest, and the whole + point of the manifest is that it can be trusted. + """ + records = list(records) + session_rows: list[list[Any]] = [] + file_rows: list[list[Any]] = [] + decision_rows: list[list[Any]] = [] + + # Attribute each decision to the checkpoint where it FIRST appeared. + # Entire stores the whole compacted session in every checkpoint, so + # raw occurrences would count one blocker once per later commit and + # make the trend monotonically increasing - the opposite of what it is + # meant to show. + first_seen = attribute_to_first_appearance(records) + + for rec in records: + for path in rec.files_touched: + if is_generated_path(path): + continue + file_rows.append( + [rec.checkpoint_id, rec.created_at, path, self._repo_key] + ) + for d in first_seen.get(rec.checkpoint_id, []): + # d.text stays on this machine. Only its shape travels: kind, + # source, confidence, size and a salted digest. The trend + # aggregate groups on `kind` and never read the prose. + sig = derive_signals(d.text, self._repo_key) + decision_rows.append( + [ + rec.checkpoint_id, + rec.created_at, + d.kind, + d.source, + d.confidence, + sig["len"], + sig["word_count"], + sig["digest"], + sig["redacted"], + self._repo_key, + ] + ) + for s in rec.sessions: + row = s.to_row() + # The stated intent is the user's own prompt. Its source and + # size are analytically useful; its words are not ours to + # upload, and the coverage aggregate only ever tested + # intent_source. + isig = derive_signals(row.get("intent", ""), self._repo_key) + session_rows.append( + [ + row["checkpoint_id"], + row["session_index"], + row["session_id"], + row["created_at"], + row["branch"], + row["agent"], + row["model"], + row["turn_count"], + row["save_step_count"], + row["prompt_count"], + row["intent_source"], + isig["len"], + isig["word_count"], + isig["digest"], + isig["redacted"], + row["files_touched_count"], + row["decision_count"], + row["input_tokens"], + row["output_tokens"], + row["cache_read_tokens"], + row["cache_creation_tokens"], + row["api_call_count"], + row["total_tokens"], + row["agent_percentage"], + row["total_lines_changed"], + rec.linked_commit, + self._repo_key, + ] + ) + + built = { + "checkpoint_sessions": session_rows, + "checkpoint_files": file_rows, + "checkpoint_decisions": decision_rows, + } + # Validate here too, so `--dry-run` fails on the same violation a real + # sync would rather than printing a manifest of rows that would be + # refused a second later. + for table, rows in built.items(): + assert_egress_safe(table, rows) + return built + + def sync(self, records: Iterable[CheckpointRecord]) -> SyncResult: + """Replace this repo's rows with the current checkpoint history. + + Idempotent by design: a DELETE for this repo followed by inserts, so + re-running after new checkpoints never double-counts. The repo key + scopes every statement, so two projects can share one workspace. + """ + reason = self.unavailable_reason() + if reason: + return SyncResult(ok=False, error=reason) + + records = list(records) + result = SyncResult() + try: + with self._connect() as conn: + with conn.cursor() as cur: + self.ensure_schema(cur) + + for table in ("checkpoint_sessions", "checkpoint_files", "checkpoint_decisions"): + cur.execute( + "DELETE FROM %s WHERE repo = ?" % self.config.table(table), + [self._repo_key], + ) + + built = self.build_rows(records) + session_rows = built["checkpoint_sessions"] + file_rows = built["checkpoint_files"] + decision_rows = built["checkpoint_decisions"] + + result.sessions_written = self._insert( + cur, "checkpoint_sessions", session_rows + ) + result.files_written = self._insert(cur, "checkpoint_files", file_rows) + result.decisions_written = self._insert( + cur, "checkpoint_decisions", decision_rows + ) + result.ok = True + except Exception as exc: # noqa: BLE001 - surface the real driver error + result.ok = False + result.error = "%s: %s" % (type(exc).__name__, exc) + return result + + def _insert(self, cursor, table: str, rows: list[list[Any]]) -> int: + """Send rows for one table. + + The egress check runs here, immediately before the statement, so it + covers every caller including any added later. A violation raises + rather than dropping the offending value: silently shipping a row + minus one column would hide the very mistake this catches. + """ + if not rows: + return 0 + assert_egress_safe(table, rows) + ncols = len(EGRESS_COLUMNS[table]) + placeholder = "(" + ",".join(["?"] * ncols) + ")" + written = 0 + # Batched multi-row INSERT: one 2X-Small serverless warehouse is the + # whole compute budget, so statement count is the thing to economise. + batch = 50 + for i in range(0, len(rows), batch): + chunk = rows[i : i + batch] + sql = "INSERT INTO %s VALUES %s" % ( + self.config.table(table), + ",".join([placeholder] * len(chunk)), + ) + flat: list[Any] = [] + for r in chunk: + flat.extend(r) + cursor.execute(sql, flat) + written += len(chunk) + return written + + # ---------------- read ---------------- + + def _query(self, name: str, sql: str, params: int = 1) -> AggregateResult: + reason = self.unavailable_reason() + if reason: + return AggregateResult(name=name, ok=False, error=reason, sql=sql) + try: + with self._connect() as conn: + with conn.cursor() as cur: + cur.execute(sql, [self._repo_key] * params) + cols = [d[0] for d in cur.description] + rows = [dict(zip(cols, r)) for r in cur.fetchall()] + return AggregateResult(name=name, ok=True, rows=rows, sql=sql) + except Exception as exc: # noqa: BLE001 + return AggregateResult( + name=name, ok=False, error="%s: %s" % (type(exc).__name__, exc), sql=sql + ) + + def coverage(self) -> AggregateResult: + # Six scalar subqueries, so six bound repo parameters. + return self._query( + "coverage", + SQL_COVERAGE.format( + sessions=self.config.table("checkpoint_sessions"), + decisions=self.config.table("checkpoint_decisions"), + ), + params=6, + ) + + def open_items_trend(self) -> AggregateResult: + return self._query( + "open_items_trend", + SQL_OPEN_ITEMS_TREND.format(decisions=self.config.table("checkpoint_decisions")), + ) + + def file_churn(self) -> AggregateResult: + return self._query( + "file_churn", SQL_FILE_CHURN.format(files=self.config.table("checkpoint_files")) + ) diff --git a/tools/checkpoint_lens/entities.py b/tools/checkpoint_lens/entities.py new file mode 100644 index 0000000000..7493346d02 --- /dev/null +++ b/tools/checkpoint_lens/entities.py @@ -0,0 +1,98 @@ +"""Entity-level change parsing for ``entire graph commit`` / ``graph diff``. + +The graph returns changes grouped by file:: + + {"base": ..., "head": ..., + "files": [{"path": ..., "status": "A", "language": "Python", + "changes": [{"type": "added", "kind": "function", + "name": "cmd_handoff", "dependents_count": 0}]}]} + +Working at this level is the whole point of using the graph instead of a text +diff: a function that merely moved reports no entity change, so it does not +show up as drift. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass +class EntityChange: + path: str + name: str + kind: str + change_type: str + language: str = "" + dependents_count: int = 0 + start_line: int = 0 + + @property + def label(self) -> str: + return "%s %s %s" % (self.change_type, self.kind, self.name) + + @property + def is_risky(self) -> bool: + """A signature change with dependents is the shape that breaks callers + silently, so it is called out separately from an ordinary edit.""" + return self.dependents_count > 0 and self.change_type in { + "signature_changed", + "removed", + "renamed", + } + + +def parse_entity_changes(data: Any) -> list[EntityChange]: + """Flatten a graph commit/diff payload into entity changes. + + Returns an empty list for any unrecognised shape rather than guessing - an + empty result is rendered by callers as "graph returned no entries", never + as "nothing changed". + """ + if not isinstance(data, dict): + return [] + files = data.get("files") + if not isinstance(files, list): + return [] + + out: list[EntityChange] = [] + for f in files: + if not isinstance(f, dict): + continue + path = str(f.get("path", "")) + language = str(f.get("language", "") or "") + changes = f.get("changes") + if not isinstance(changes, list): + continue + for c in changes: + if not isinstance(c, dict): + continue + out.append( + EntityChange( + path=path, + name=str(c.get("name", "") or ""), + kind=str(c.get("kind", "") or ""), + change_type=str(c.get("type", "") or ""), + language=language, + dependents_count=int(c.get("dependents_count", 0) or 0), + start_line=int(c.get("after_start_line", 0) or 0), + ) + ) + return out + + +def touched_paths(changes: list[EntityChange]) -> list[str]: + seen: list[str] = [] + for c in changes: + if c.path and c.path not in seen: + seen.append(c.path) + return seen + + +def risky(changes: list[EntityChange]) -> list[EntityChange]: + return sorted( + [c for c in changes if c.is_risky], + key=lambda c: c.dependents_count, + reverse=True, + ) diff --git a/tools/checkpoint_lens/graph.py b/tools/checkpoint_lens/graph.py new file mode 100644 index 0000000000..954a8024a2 --- /dev/null +++ b/tools/checkpoint_lens/graph.py @@ -0,0 +1,251 @@ +"""GraphClient - a thin, honest wrapper over the ``entire graph`` plugin. + +Two rules this module enforces, both from the Graph operating guide: + +1. Graph output is *evidence*, not an oracle. Every result carries the exact + command that produced it (:attr:`GraphResult.command`) so a reader can rerun + it and verify the finding against source. +2. A missing or failing graph never fabricates an answer. It degrades to + ``ok=False`` with the real stderr, and callers render that as "unavailable" + rather than as "no impact" - the two mean very different things to someone + deciding whether a change is safe. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from dataclasses import dataclass, field +from typing import Any + +DEFAULT_TIMEOUT = 90 + + +@dataclass +class GraphResult: + """One graph invocation: what was asked, what came back, and whether it + can be trusted.""" + + command: list[str] + ok: bool + data: Any = None + text: str = "" + error: str = "" + + @property + def command_str(self) -> str: + return " ".join(self.command) + + +@dataclass +class ImpactSummary: + """Blast radius for one symbol, flattened for reporting.""" + + symbol: str + callers: list[str] = field(default_factory=list) + callees: list[str] = field(default_factory=list) + type_consumers: list[str] = field(default_factory=list) + cochange_files: list[str] = field(default_factory=list) + ok: bool = True + error: str = "" + command: str = "" + + @property + def blast_radius(self) -> int: + return len(self.callers) + len(self.type_consumers) + + +class GraphClient: + def __init__(self, repo: str = ".", timeout: int = DEFAULT_TIMEOUT) -> None: + self.repo = repo + self.timeout = timeout + self._available: bool | None = None + + # ---------------- plumbing ---------------- + + def available(self) -> bool: + """True when the graph plugin can actually answer. Cached - the check + shells out, and every command path asks.""" + if self._available is None: + if shutil.which("entire") is None: + self._available = False + else: + self._available = self._run(["graph", "version"]).ok + return self._available + + def _run(self, args: list[str]) -> GraphResult: + cmd = ["entire", *args] + try: + proc = subprocess.run( + cmd, capture_output=True, timeout=self.timeout, check=False + ) + except (subprocess.TimeoutExpired, OSError) as exc: + return GraphResult(command=cmd, ok=False, error=str(exc)) + + out = proc.stdout.decode("utf-8", "replace") + err = proc.stderr.decode("utf-8", "replace").strip() + if proc.returncode != 0: + return GraphResult(command=cmd, ok=False, text=out, error=err or "exit %d" % proc.returncode) + + data: Any = None + stripped = out.strip() + if stripped.startswith("{") or stripped.startswith("["): + try: + data = json.loads(stripped) + except json.JSONDecodeError: + data = None + return GraphResult(command=cmd, ok=True, data=data, text=out) + + # ---------------- graph commands ---------------- + + def capabilities(self) -> GraphResult: + return self._run(["graph", "capabilities", "--json"]) + + def search(self, query: str, profile: str = "full", limit: int = 10) -> GraphResult: + """Ranked code regions for a plain-language query. + + The result-count flag is ``--top-k`` (not ``--limit``); the wrong + spelling makes the plugin fail and every requirement then reports + UNVERIFIED, which is indistinguishable at a glance from a genuinely + unanswerable query. + """ + return self._run( + [ + "graph", "search", + "--repo", self.repo, + "--profile", profile, + "--query", query, + "--top-k", str(limit), + "--format", "json", + "--max-context-bytes", "4096", + ] + ) + + def impact(self, symbol: str, depth: int = 2, limit: int = 10) -> ImpactSummary: + res = self._run( + [ + "graph", "impact", + "--repo", self.repo, + "--symbol", symbol, + "--depth", str(depth), + "--limit", str(limit), + "--format", "json", + ] + ) + summary = ImpactSummary(symbol=symbol, ok=res.ok, error=res.error, command=res.command_str) + if not res.ok: + return summary + payload = res.data if isinstance(res.data, dict) else {} + summary.callers = _names(payload, "callers", "callers_direct", "direct_callers") + summary.callees = _names(payload, "callees") + summary.type_consumers = _names(payload, "type_consumers", "uses_type") + summary.cochange_files = _names(payload, "cochange", "co_change", "cochange_files") + if not payload and res.text: + # Text mode fallback: keep the prose so the report can still show + # the evidence, even when the JSON shape is not what we expected. + summary.error = "" + summary.callers = [] + return summary + + def commit_entities(self, commitish: str, max_seconds: int = 120) -> GraphResult: + """Entity-level change list for a commit vs its first parent. + + Note the interface: ``rev`` is positional and the flag is ``--json`` + (not ``--commit``/``--format``). Passing the wrong spelling makes the + plugin exit 0 with "commit accepts at most one revision" on stdout, + which is why every result here is shape-checked rather than trusted. + """ + return self._run( + [ + "graph", "commit", commitish, + "--repo", self.repo, + "--max-seconds", str(max_seconds), + "--json", + ] + ) + + def diff(self, base: str, head: str) -> GraphResult: + """Entity-level change list between two refs. + + This is the drift primitive: it reports which *entities* changed, so a + function that merely moved does not read as drift the way a raw text + diff would. + """ + return self._run( + ["graph", "diff", "--repo", self.repo, "--base", base, "--head", head, "--json"] + ) + + def checkpoint(self, checkpoint_id: str) -> GraphResult: + """Analyze the commit carrying this checkpoint's trailer.""" + return self._run( + ["graph", "checkpoint", "--repo", self.repo, "--id", checkpoint_id, "--format", "json"] + ) + + def verify( + self, test_command: str, baseline: str, timeout: int = 900 + ) -> tuple[GraphResult, bool]: + """Run a test command and return an adjudicated verdict. + + Deliberately uses ``entire graph verify`` rather than running the tests + ourselves: it reports *which tests changed state* - newly passing, + newly failing, or already failing beforehand - instead of dumping + runner output. That distinction is the whole point when the question is + "does this change break anything", because a test that was already red + is not evidence against this change. + + The verifier requires a baseline. When none exists yet this records one + and returns ``recorded=True``: the result is then a *state*, not a + delta, and the caller must say so rather than presenting a first run as + proof that nothing regressed. + """ + flag = "--pre-edit-baseline" if os.path.isfile(baseline) else "--record-baseline" + previous, self.timeout = self.timeout, timeout + try: + res = self._run( + [ + "graph", "verify", + "--repo", self.repo, + "--test", test_command, + flag, baseline, + ] + ) + finally: + self.timeout = previous + return res, flag == "--record-baseline" + + +def _names(payload: dict[str, Any], *keys: str) -> list[str]: + """Pull a list of symbol/file names out of whichever key the graph used. + + The graph's JSON shape varies by section and version, so this accepts a few + spellings and tolerates both bare strings and objects. + """ + for key in keys: + value = payload.get(key) + if not value: + continue + out: list[str] = [] + if isinstance(value, dict): + value = value.get("items") or value.get("entries") or [] + if isinstance(value, list): + for item in value: + if isinstance(item, str): + out.append(item) + elif isinstance(item, dict): + name = ( + item.get("symbol") + or item.get("name") + or item.get("file") + or item.get("path") + ) + if name: + loc = item.get("file") or item.get("path") + if loc and loc != name: + out.append("%s (%s)" % (name, loc)) + else: + out.append(str(name)) + if out: + return out + return [] diff --git a/tools/checkpoint_lens/html.py b/tools/checkpoint_lens/html.py new file mode 100644 index 0000000000..0abd9a322b --- /dev/null +++ b/tools/checkpoint_lens/html.py @@ -0,0 +1,390 @@ +"""Self-contained HTML reports. + +One file, no server, no CDN, no external fonts - everything inline. That is a +deliberate constraint: the report has to open from a USB stick or an email +attachment on a judge's or a colleague's machine, and it has to keep working +when the network, the Databricks warehouse, or the graph plugin does not. + +Anything that could not be computed is rendered as an explicit "unavailable" +panel rather than being omitted, because a silently missing section reads as a +clean bill of health. + +Above all of that sits ONE completeness banner, the same verdict the terminal +prints, placed before the stat grid. The per-panel notes say which piece is +missing; the banner is the single thing a reader cannot miss, and it is what +stops a partial reconstruction from being read as a full one. +""" + +from __future__ import annotations + +import html as _html +from typing import Any + +from .models import CheckpointRecord +from .report import KIND_LABEL, SOURCE_LABEL, intent_provenance + +CSS = """ +:root{ + --bg:#f7f7f5; --panel:#ffffff; --ink:#16161a; --muted:#5c5f66; + --line:#e3e3df; --accent:#2f5fd0; --shadow:0 1px 2px rgba(0,0,0,.05); + --blocker:#b3261e; --risk:#a8560c; --open:#8a6d00; --rejected:#6b4ea8; + --decision:#1d6f42; --rationale:#3a6b8a; --assumption:#5c5f66; +} +@media (prefers-color-scheme:dark){ + :root{ + --bg:#141416; --panel:#1c1c1f; --ink:#ececef; --muted:#a2a5ad; + --line:#2c2c31; --accent:#8fb0ff; --shadow:none; + --blocker:#ff8a80; --risk:#ffb870; --open:#ffd95e; --rejected:#c3a6ff; + --decision:#7ee0a5; --rationale:#8fc7e8; --assumption:#a2a5ad; + } +} +*{box-sizing:border-box} +body{margin:0;background:var(--bg);color:var(--ink); + font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif} +.wrap{max-width:960px;margin:0 auto;padding:32px 20px 64px} +header{border-bottom:2px solid var(--ink);padding-bottom:16px;margin-bottom:28px} +h1{font-size:24px;margin:0 0 6px;letter-spacing:-.01em} +.sub{color:var(--muted);font-size:14px} +h2{font-size:13px;text-transform:uppercase;letter-spacing:.09em; + color:var(--muted);margin:32px 0 12px;font-weight:600} +.panel{background:var(--panel);border:1px solid var(--line);border-radius:8px; + padding:16px 18px;box-shadow:var(--shadow)} +.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px} +.stat{background:var(--panel);border:1px solid var(--line);border-radius:8px;padding:12px 14px} +.stat .n{font-size:22px;font-weight:650;letter-spacing:-.02em} +.stat .l{font-size:12px;color:var(--muted);margin-top:2px} +.prov{font-size:12px;color:var(--muted);font-style:italic;margin-top:6px} +.item{border-left:3px solid var(--line);padding:8px 0 8px 14px;margin:12px 0} +.item .tag{display:inline-block;font-size:11px;font-weight:700;letter-spacing:.06em; + padding:1px 7px;border-radius:3px;border:1px solid currentColor;margin-bottom:6px} +.item .src{font-size:11.5px;color:var(--muted);margin-top:5px} +.k-blocker{border-left-color:var(--blocker)} .k-blocker .tag{color:var(--blocker)} +.k-risk{border-left-color:var(--risk)} .k-risk .tag{color:var(--risk)} +.k-open_question{border-left-color:var(--open)} .k-open_question .tag{color:var(--open)} +.k-rejected{border-left-color:var(--rejected)} .k-rejected .tag{color:var(--rejected)} +.k-decision{border-left-color:var(--decision)} .k-decision .tag{color:var(--decision)} +.k-rationale{border-left-color:var(--rationale)} .k-rationale .tag{color:var(--rationale)} +.k-assumption{border-left-color:var(--assumption)} .k-assumption .tag{color:var(--assumption)} +code,pre{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12.5px} +pre{background:var(--panel);border:1px solid var(--line);border-radius:6px; + padding:10px 12px;overflow-x:auto;margin:8px 0} +ul.files{list-style:none;padding:0;margin:0;columns:2;column-gap:24px} +ul.files li{font-family:ui-monospace,monospace;font-size:12.5px;padding:2px 0; + break-inside:avoid;color:var(--muted)} +table{border-collapse:collapse;width:100%;font-size:13px} +th,td{text-align:left;padding:7px 10px;border-bottom:1px solid var(--line)} +th{font-size:11px;text-transform:uppercase;letter-spacing:.06em;color:var(--muted)} +td.num{text-align:right;font-variant-numeric:tabular-nums} +.verdict{font-weight:650;font-size:12px} +.v-IMPLEMENTED{color:var(--decision)} .v-PARTIAL{color:var(--risk)} +.v-MISSING{color:var(--blocker)} .v-UNVERIFIED{color:var(--muted)} +.warn{border-left:3px solid var(--risk);background:var(--panel);padding:12px 14px; + border-radius:0 6px 6px 0;margin:10px 0;font-size:13.5px} +.na{color:var(--muted);font-style:italic} +.banner{border:2px solid var(--decision);border-radius:8px;padding:14px 16px; + margin:0 0 24px;background:var(--panel)} +.banner.partial{border-color:var(--risk)} +.banner .verdict-line{font-size:16px;font-weight:700;letter-spacing:-.01em;margin-bottom:4px} +.banner.partial .verdict-line{color:var(--risk)} +.banner .lede{font-size:13px;color:var(--muted);margin-bottom:10px} +.banner ul{list-style:none;padding:0;margin:0} +.banner li{font-size:13px;padding:3px 0;display:flex;gap:8px;align-items:baseline} +.banner .st{flex:0 0 92px;font-size:10.5px;font-weight:700;letter-spacing:.06em; + text-transform:uppercase;font-family:ui-monospace,monospace} +.banner .st-available{color:var(--decision)} +.banner .st-missing{color:var(--blocker)} +.banner .st-redacted{color:var(--risk)} +.banner .why{color:var(--muted)} +footer{margin-top:44px;padding-top:16px;border-top:1px solid var(--line); + font-size:12px;color:var(--muted)} +.scroll{overflow-x:auto} +""" + + +def esc(text: Any) -> str: + return _html.escape(str(text if text is not None else "")) + + +def _stat(n: Any, label: str) -> str: + return '
%s
%s
' % ( + esc(n), + esc(label), + ) + + +def _decisions_block(decisions: list[Any], limit: int = 40) -> str: + if not decisions: + return '
No decision context recovered from this checkpoint.
' + order = ["blocker", "open_question", "risk", "rejected", "decision", "assumption", "rationale"] + by_kind: dict[str, list[Any]] = {} + for d in decisions: + by_kind.setdefault(d.kind, []).append(d) + out: list[str] = [] + shown = 0 + for kind in order: + for d in by_kind.get(kind, []): + if shown >= limit: + break + out.append( + '
%s' + '
%s
source: %s · confidence: %s
' + % ( + esc(kind), + esc(KIND_LABEL.get(kind, kind.upper())), + esc(d.text), + esc(SOURCE_LABEL.get(d.source, d.source)), + esc(d.confidence), + ) + ) + shown += 1 + if shown < len(decisions): + out.append('

%d more in the JSON output.

' % (len(decisions) - shown)) + return "\n".join(out) + + +def _trend_svg(rows: list[dict[str, Any]]) -> str: + """A bar chart of unresolved items per checkpoint, drawn as inline SVG. + + Hand-rolled rather than pulled from a charting CDN: the file has to render + with no network. Two checkpoints or fewer is not a trend, and the caller is + told so rather than being shown a line through two points. + """ + if not rows: + return '
No trend data.
' + vals = [int(r.get("unresolved_total") or 0) for r in rows] + peak = max(vals) or 1 + w, h, pad, gap = 900, 170, 28, 10 + n = len(vals) + bar = max(14, (w - 2 * pad - gap * (n - 1)) // n) + bars: list[str] = [] + for i, v in enumerate(vals): + bh = int((h - 2 * pad) * (v / peak)) + x = pad + i * (bar + gap) + y = h - pad - bh + label = esc(str(rows[i].get("checkpoint_id", ""))[:6]) + bars.append( + '' + '%d' + '%s' + % ( + x, y, bar, max(bh, 2), 0.35 + 0.65 * (v / peak), + x + bar // 2, max(y - 5, 12), v, + x + bar // 2, h - pad + 14, label, + ) + ) + return ( + '
' + '%s
' + % (w, h, h, "".join(bars)) + ) + + +def _completeness_banner(comp: Any) -> str: + """The same single verdict the terminal prints, first in the document. + + It sits above the stat grid on purpose: those big numbers are the most + confident-looking thing on the page, and a reader must know what they are + computed from before reading them. + """ + if comp is None: + return "" + cls = "banner" if comp.is_complete else "banner partial" + parts = ['
' % cls] + if comp.is_complete: + parts.append( + '
Context: COMPLETE
' + '
All %d inputs behind this report were readable.
' + % len(comp.inputs) + ) + else: + parts.append( + '
Context: PARTIAL
' + '
%d of %d inputs are missing or redacted. Everything ' + "below is reconstructed from the rest, and the absence of a finding " + "is not evidence that there is nothing to find.
" + % (len(comp.degraded), len(comp.inputs)) + ) + parts.append("
    ") + for i in comp.inputs: + label = {"available": "ok", "missing": "unavailable", "redacted": "redacted"}.get( + i.status, i.status + ) + why = (" — " + esc(i.detail)) if i.detail else "" + parts.append( + '
  • %s%s%s
    • ' + % (esc(i.status), esc(label), esc(i.name), why) + ) + parts.append("
") + return "".join(parts) + + +def render_handoff( + rec: CheckpointRecord, + history: list[CheckpointRecord], + warnings: list[str], + graph_lines: list[str] | None = None, + aggregates: dict[str, Any] | None = None, + comp: Any = None, +) -> str: + agg = aggregates or {} + sess = rec.sessions[0] if rec.sessions else None + + parts: list[str] = [] + parts.append("

Checkpoint Lens — Handoff Brief

") + parts.append( + '
%s · %s · branch %s
' + % (esc(rec.checkpoint_id), esc(rec.created_at or "unknown time"), esc(rec.branch or "?")) + ) + + parts.append(_completeness_banner(comp)) + + parts.append('
') + parts.append(_stat(len(rec.files_touched), "files touched")) + parts.append(_stat(len(rec.all_decisions()), "decisions recovered")) + if sess: + parts.append(_stat("%.0f%%" % sess.attribution.agent_percentage, "agent-written")) + parts.append(_stat(format(sess.attribution.total_lines_changed, ","), "lines changed")) + parts.append(_stat(format(rec.token_usage.total, ","), "tokens")) + parts.append("
") + + parts.append("

Stated intent

") + parts.append("
%s
" % esc(rec.intent[:2000] or "No intent recoverable.")) + parts.append('
Provenance: %s
' % esc(intent_provenance(rec.intent_source))) + + if rec.linked_commit: + parts.append("

Linked commit

%s — %s
" + % (esc(rec.linked_commit[:12]), esc(rec.linked_subject))) + + parts.append("

Decisions, risks and open questions

") + parts.append('

Recovered verbatim from checkpoint context — never paraphrased or generated.

') + parts.append(_decisions_block(rec.all_decisions())) + + if graph_lines: + parts.append("

Graph impact (blast radius)

") + parts.append("
%s
" % esc("\n".join(graph_lines))) + + trend = agg.get("trend") + if trend is not None: + parts.append("

Unresolved context over time (Databricks)

") + if trend: + parts.append(_trend_svg(trend)) + if len(trend) < 3: + parts.append('

Only %d checkpoints — too few to read as a trend.

' % len(trend)) + else: + parts.append('
Databricks returned no rows.
') + elif agg.get("unavailable"): + parts.append("

Unresolved context over time (Databricks)

") + parts.append('
Cross-session analytics unavailable: %s
' + "Single-checkpoint sections above are unaffected.
" % esc(agg["unavailable"])) + + parts.append("

Files touched

    ") + parts.extend("
  • %s
    • " % esc(f) for f in rec.files_touched) + parts.append("
") + + if len(history) > 1: + parts.append("

Checkpoint history

") + parts.append("") + for r in history: + parts.append( + "" + % (esc(r.checkpoint_id), esc((r.created_at or "")[:19]), esc(r.linked_subject or "(unlinked)")) + ) + parts.append("
CheckpointWhenCommit
%s%s%s
") + + if warnings: + parts.append("

Warnings — this view may be incomplete

") + parts.extend('
%s
' % esc(w) for w in warnings) + + parts.append( + "
Generated by entire lens from real Entire Checkpoints. " + "Every figure is derived from committed checkpoint data; nothing is synthesised. " + "Graph findings are evidence, not proof — rerun the printed commands to verify." + "
" + ) + return _document("Checkpoint Lens - Handoff", "".join(parts)) + + +def render_drift( + baseline: CheckpointRecord, + head: CheckpointRecord, + findings: list[Any], + entity_changes: list[Any], + diff_command: str, + warnings: list[str], + comp: Any = None, +) -> str: + from .requirements import IMPLEMENTED, MISSING, PARTIAL, UNVERIFIED, VERDICT_NOTE + + counts: dict[str, int] = {} + for f in findings: + counts[f.verdict] = counts.get(f.verdict, 0) + 1 + total = len(findings) or 1 + done = counts.get(IMPLEMENTED, 0) + + parts: list[str] = [] + parts.append("

Checkpoint Lens — Drift Report

") + parts.append('
Plan %s → current %s
' + % (esc(baseline.checkpoint_id), esc(head.checkpoint_id))) + + parts.append(_completeness_banner(comp)) + + parts.append('
') + parts.append(_stat("%.0f%%" % (100.0 * done / total), "requirement coverage")) + parts.append(_stat(len(findings), "requirements extracted")) + parts.append(_stat(counts.get(MISSING, 0) + counts.get(PARTIAL, 0), "still open")) + parts.append(_stat(len(entity_changes), "entity changes")) + parts.append("
") + + parts.append("

Plan vs implementation

") + parts.append('
') + parts.append("") + for f in findings: + parts.append( + '' + % ( + esc(f.requirement.text), + esc(f.verdict), + esc(f.verdict), + esc((f.evidence[0] if f.evidence else "")[:90]), + ) + ) + parts.append("
RequirementVerdictEvidence
%s%s%s
") + parts.append('

%s

' % esc(VERDICT_NOTE[MISSING])) + + parts.append("

Entity-level change since the plan

") + if diff_command: + parts.append("
$ %s
" % esc(diff_command)) + if entity_changes: + parts.append( + '

Entity-level, so a function that merely moved is not counted as drift.

' + ) + parts.append('
') + parts.append("") + for c in entity_changes[:60]: + parts.append( + "" + % (esc(c.label), esc(c.path), esc(c.dependents_count or "")) + ) + parts.append("
ChangeFileDependents
%s%s%s
") + else: + parts.append('
No entity changes reported.
') + + if warnings: + parts.append("

Warnings

") + parts.extend('
%s
' % esc(w) for w in warnings) + + parts.append( + "
MISSING means no evidence was found, not that the work is absent. " + "Verify each open item against source before acting.
" + ) + return _document("Checkpoint Lens - Drift", "".join(parts)) + + +def _document(title: str, body: str) -> str: + return ( + "" + "" + "%s
%s
" + % (esc(title), CSS, body) + ) diff --git a/tools/checkpoint_lens/models.py b/tools/checkpoint_lens/models.py new file mode 100644 index 0000000000..e4e8646080 --- /dev/null +++ b/tools/checkpoint_lens/models.py @@ -0,0 +1,216 @@ +"""Structured records parsed out of real Entire Checkpoints. + +These dataclasses are the single schema shared by every consumer: the terminal +and HTML reports, and the Databricks Delta table. Keeping one schema is what +lets the same record be rendered locally and aggregated across sessions. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field, asdict +from typing import Any + + +# Entire's own redaction pipeline replaces a detected secret with this marker +# before the transcript is ever committed, so it is what a redacted checkpoint +# actually looks like on disk. Both the egress layer and the completeness +# banner need to recognise it: text carrying the marker is content we do NOT +# have, and rendering it as though it were recovered content is the exact +# "incomplete presented as complete" failure this tool exists to prevent. +REDACTION_MARKER = re.compile(r"(?:\[)?\bREDACTED\b(?:\])?") + + +def has_redaction_marker(text: str) -> bool: + """True when Entire's redaction pipeline removed content from this text.""" + return bool(text) and bool(REDACTION_MARKER.search(text)) + + +# How a session's "stated intent" was obtained. Surfaced in every report so a +# reader can tell recovered context from synthesised context (responsible-use +# rule: label synthetic or derived content). +INTENT_FROM_SUMMARY = "checkpoint_summary" +INTENT_FROM_FIRST_PROMPT = "first_user_prompt" +INTENT_FROM_TRANSCRIPT = "first_user_message_in_transcript" +INTENT_UNAVAILABLE = "unavailable" + + +@dataclass +class TokenUsage: + input_tokens: int = 0 + cache_creation_tokens: int = 0 + cache_read_tokens: int = 0 + output_tokens: int = 0 + api_call_count: int = 0 + + @classmethod + def from_dict(cls, d: dict[str, Any] | None) -> "TokenUsage": + d = d or {} + return cls( + input_tokens=int(d.get("input_tokens", 0) or 0), + cache_creation_tokens=int(d.get("cache_creation_tokens", 0) or 0), + cache_read_tokens=int(d.get("cache_read_tokens", 0) or 0), + output_tokens=int(d.get("output_tokens", 0) or 0), + api_call_count=int(d.get("api_call_count", 0) or 0), + ) + + @property + def total(self) -> int: + return ( + self.input_tokens + + self.cache_creation_tokens + + self.cache_read_tokens + + self.output_tokens + ) + + +@dataclass +class Attribution: + """Who wrote the code in this session - agent vs human.""" + + agent_lines: int = 0 + human_added: int = 0 + human_modified: int = 0 + human_removed: int = 0 + total_lines_changed: int = 0 + agent_percentage: float = 0.0 + + @classmethod + def from_dict(cls, d: dict[str, Any] | None) -> "Attribution": + d = d or {} + return cls( + agent_lines=int(d.get("agent_lines", 0) or 0), + human_added=int(d.get("human_added", 0) or 0), + human_modified=int(d.get("human_modified", 0) or 0), + human_removed=int(d.get("human_removed", 0) or 0), + total_lines_changed=int(d.get("total_lines_changed", 0) or 0), + agent_percentage=float(d.get("agent_percentage", 0.0) or 0.0), + ) + + +# Where a recovered decision came from, in descending order of authority. A +# commit message is a deliberate, edited record; transcript prose is a +# by-product of working. Both are verbatim, but they are not equally reliable, +# and the report says which is which. +SOURCE_SUMMARY = "checkpoint_summary" +SOURCE_COMMIT = "commit_message" +SOURCE_TRANSCRIPT = "transcript" + +SOURCE_CONFIDENCE = { + SOURCE_SUMMARY: "high", + SOURCE_COMMIT: "high", + SOURCE_TRANSCRIPT: "medium", +} + + +@dataclass +class Decision: + """A decision, rejected option, assumption, risk or blocker recovered from + a checkpoint. `kind` is the classifier; `text` is verbatim prose, never + paraphrased, so a reader can audit it against the source.""" + + kind: str + text: str + speaker: str = "assistant" + source: str = SOURCE_TRANSCRIPT + + @property + def confidence(self) -> str: + return SOURCE_CONFIDENCE.get(self.source, "medium") + + +@dataclass +class SessionRecord: + checkpoint_id: str + session_index: int + session_id: str = "" + created_at: str = "" + branch: str = "" + agent: str = "" + model: str = "" + turn_count: int = 0 + save_step_count: int = 0 + prompts: list[str] = field(default_factory=list) + intent: str = "" + intent_source: str = INTENT_UNAVAILABLE + files_touched: list[str] = field(default_factory=list) + decisions: list[Decision] = field(default_factory=list) + token_usage: TokenUsage = field(default_factory=TokenUsage) + attribution: Attribution = field(default_factory=Attribution) + # Which of this session's inputs were actually readable. These exist so + # that "we read it and there was nothing" and "we could not read it" are + # different states everywhere downstream. Before they existed, an absent + # metadata blob became an empty dict and rendered as `0 turns / unknown + # agent`, which is a confident-looking answer to a question we had not + # asked anything about. + metadata_available: bool = False + prompt_available: bool = False + transcript_available: bool = False + redacted_content: bool = False + + def to_row(self) -> dict[str, Any]: + """Flatten to one Delta-table row. Nested structs are flattened because + the aggregate SQL groups and sums over these columns directly.""" + return { + "checkpoint_id": self.checkpoint_id, + "session_index": self.session_index, + "session_id": self.session_id, + "created_at": self.created_at, + "branch": self.branch, + "agent": self.agent, + "model": self.model, + "turn_count": self.turn_count, + "save_step_count": self.save_step_count, + "intent": self.intent, + "intent_source": self.intent_source, + "prompt_count": len(self.prompts), + "files_touched": list(self.files_touched), + "files_touched_count": len(self.files_touched), + "decision_count": len(self.decisions), + "input_tokens": self.token_usage.input_tokens, + "output_tokens": self.token_usage.output_tokens, + "cache_read_tokens": self.token_usage.cache_read_tokens, + "cache_creation_tokens": self.token_usage.cache_creation_tokens, + "api_call_count": self.token_usage.api_call_count, + "total_tokens": self.token_usage.total, + "agent_percentage": self.attribution.agent_percentage, + "total_lines_changed": self.attribution.total_lines_changed, + } + + +@dataclass +class CheckpointRecord: + checkpoint_id: str + ref: str = "" + checkpoint_commit: str = "" + linked_commit: str = "" + linked_subject: str = "" + created_at: str = "" + branch: str = "" + files_touched: list[str] = field(default_factory=list) + token_usage: TokenUsage = field(default_factory=TokenUsage) + sessions: list[SessionRecord] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def intent(self) -> str: + for s in self.sessions: + if s.intent: + return s.intent + return "" + + @property + def intent_source(self) -> str: + for s in self.sessions: + if s.intent: + return s.intent_source + return INTENT_UNAVAILABLE + + def all_decisions(self) -> list[Decision]: + out: list[Decision] = [] + for s in self.sessions: + out.extend(s.decisions) + return out + + def to_dict(self) -> dict[str, Any]: + return asdict(self) diff --git a/tools/checkpoint_lens/reader.py b/tools/checkpoint_lens/reader.py new file mode 100644 index 0000000000..97f0e890c9 --- /dev/null +++ b/tools/checkpoint_lens/reader.py @@ -0,0 +1,658 @@ +"""CheckpointReader - parses real Entire Checkpoints into structured records. + +This repo uses the git-refs checkpoint backend, so checkpoints live under +``refs/entire/checkpoints//`` rather than on the legacy +``entire/checkpoints/v1`` branch. Each ref points at a commit whose tree is:: + + metadata.json root manifest (sessions[], files_touched, ...) + /metadata.json per-session metadata (attribution, tokens) + /prompt.txt every user prompt, separator-joined + /transcript.jsonl compact transcript + /full.jsonl full transcript (large; not read by default) + +Reads go through git plumbing (``git cat-file``) against the checkpoint tree, so +nothing is checked out and the working tree is never touched. +""" + +from __future__ import annotations + +import json +import re +import subprocess +from typing import Any, Iterable + +from .models import ( + has_redaction_marker, + INTENT_FROM_FIRST_PROMPT, + INTENT_FROM_SUMMARY, + INTENT_FROM_TRANSCRIPT, + INTENT_UNAVAILABLE, + Attribution, + CheckpointRecord, + Decision, + SOURCE_COMMIT, + SOURCE_TRANSCRIPT, + SessionRecord, + TokenUsage, +) + +CHECKPOINT_REF_PREFIX = "refs/entire/checkpoints/" +CHECKPOINT_TRAILER = "Entire-Checkpoint:" +PROMPT_SEPARATOR = re.compile(r"\n-{3,}\n") + +# Markers that classify a line of transcript prose as decision context. This is +# the difference between "what changed" (git) and "why it changed" (checkpoint). +# Order is significant: the first matching kind wins, so the most specific +# and most consequential classifications are tried first. +# +# "instead of" and "rather than" are deliberately NOT rejection markers. They +# are ordinary English connectives that appear in most technical prose, and +# including them classified plain statements of fact as rejected options. A +# rejection has to be stated as one. +# Each kind is a regex requiring the marker to be *predicated of this work*, +# not merely mentioned. +# +# Substring matching on bare nouns was tried first and does not work: "risk" +# matched "file-churn/risk score" (a column name), and "abandoned" matched a +# sentence *defining* what checkpoints preserve ("what was attempted and +# abandoned"). Both were reported as findings about the project. A marker has +# to appear as a claim - "we abandoned", "the risk is" - to count. +DECISION_MARKERS: list[tuple[str, "re.Pattern[str]"]] = [ + ( + "blocker", + re.compile( + r"\b(hit a blocker|blocked on|is a blocker|blocker:|cannot proceed|" + r"could not proceed|is not possible|does not exist in this repo)\b", + re.IGNORECASE, + ), + ), + ( + "rejected", + re.compile( + r"\b(reject(s|ed|ing)?|ruled out|decided against|abandoned|discarded|" + r"(we|i) are not going to)\b", + re.IGNORECASE, + ), + ), + ( + "decision", + re.compile( + r"\b((we|i) decided|decided to|(we|i) chose|chose to|opted (for|to)|" + r"going with|(we|i) will use|settled on|the decision (is|was)|" + # Engineering prose states decisions passively as often as it states + # them agentively: "deliberately a transparent classifier", "chosen + # over an LLM call", "in favour of Python". Requiring "we decided" + # alone missed most real decisions in commit messages. + r"deliberately|by design|on purpose|chosen over|in favou?r of|" + r"we prefer|is preferred over)\b", + re.IGNORECASE, + ), + ), + ( + "risk", + re.compile( + r"\b(the risk (is|here|being)|risks? that|at risk of|is risky|" + r"could break|would break|regression|is unsafe|is fragile|" + r"is dangerous|biggest .{0,20}risk|quality risk)\b", + re.IGNORECASE, + ), + ), + ( + "open_question", + re.compile( + r"\b(still unresolved|remains unresolved|remains open|still needs? to|" + r"not yet decided|not yet confirmed|to be decided|still not sure|" + r"undecided|open question:|not yet implemented|still missing)\b", + re.IGNORECASE, + ), + ), + ( + "assumption", + re.compile( + r"\b((we|i) assume|assuming that|the assumption (is|was)|presumably|" + r"on the assumption)\b", + re.IGNORECASE, + ), + ), + # Last, so any more specific kind wins. This is the catch-all for stated + # *reasoning* - the causal prose that says why something is the way it is. + # It is the single most abundant form the "why" actually takes in commit + # messages and agent explanations ("chosen so that ...", "idempotent, so + # re-running never double-counts"), and omitting it lost most of the + # product's own subject matter. + ( + "rationale", + re.compile( + r"\b(because|so that|the reason|which is why|on the grounds that|" + r"rather than|instead of|in order to|otherwise)\b", + re.IGNORECASE, + ), + ), +] + +# A line that is mostly capitals is a section heading, not a sentence. Commit +# messages in this project use them freely, and they classified as findings. +def _is_heading(sentence: str) -> bool: + letters = [c for c in sentence if c.isalpha()] + if len(letters) < 8: + return False + upper = sum(1 for c in letters if c.isupper()) + return upper / len(letters) > 0.7 + +# Markdown emphasis and inline-code fences are noise in a terminal report. +MARKUP = re.compile(r"\*\*|__|`") + +MIN_DECISION_LEN = 30 +MAX_DECISION_LEN = 400 + +# git log record/field separators - ASCII unit/record separator, chosen because +# they cannot occur in a commit subject or body. +_FIELD_SEP = "\x1f" +_RECORD_SEP = "\x1e" + + +class GitError(RuntimeError): + pass + + +class CheckpointReader: + """Reads checkpoints out of a repository. Generic: takes any repo path.""" + + def __init__(self, repo: str = ".") -> None: + self.repo = repo + self._warnings: list[str] = [] + + # ---------------- git plumbing ---------------- + + def _git(self, *args: str) -> str: + proc = subprocess.run( + ["git", "-C", self.repo, *args], + capture_output=True, + check=False, + ) + if proc.returncode != 0: + msg = proc.stderr.decode("utf-8", "replace").strip() + raise GitError("git " + " ".join(args) + ": " + msg) + return proc.stdout.decode("utf-8", "replace") + + def _blob(self, tree: str, path: str) -> str | None: + """Read one path out of a checkpoint tree. A missing path is not fatal - + a checkpoint may legitimately omit a transcript or prompt file.""" + path = path.lstrip("/") + try: + return self._git("cat-file", "-p", tree + ":" + path) + except GitError: + return None + + @property + def warnings(self) -> list[str]: + return list(self._warnings) + + # ---------------- discovery ---------------- + + def list_refs(self) -> list[tuple[str, str, str]]: + """Return (checkpoint_id, ref, commit_sha), oldest first. + + Checkpoint IDs are ULIDs, which sort lexicographically by creation time, + so a plain sort is a true chronological ordering. + """ + out = self._git( + "for-each-ref", "--format=%(objectname)\t%(refname)", CHECKPOINT_REF_PREFIX + ) + rows: list[tuple[str, str, str]] = [] + for line in out.splitlines(): + if not line.strip(): + continue + sha, _, ref = line.partition("\t") + cid = ref.rsplit("/", 1)[-1] + rows.append((cid, ref, sha)) + rows.sort(key=lambda r: r[0]) + return rows + + def commit_links(self) -> dict[str, tuple[str, str]]: + """Map checkpoint_id -> (repo commit sha, subject) via the + ``Entire-Checkpoint:`` commit trailer the CLI writes on every + checkpointed commit.""" + links: dict[str, tuple[str, str]] = {} + fmt = "--format=%H" + _FIELD_SEP + "%s" + _FIELD_SEP + "%b" + _RECORD_SEP + try: + log = self._git("log", fmt, "--all") + except GitError as exc: + self._warnings.append("could not scan commit trailers: " + str(exc)) + return links + for entry in log.split(_RECORD_SEP): + entry = entry.strip("\n") + if not entry: + continue + parts = entry.split(_FIELD_SEP) + if len(parts) < 3: + continue + sha, subject, body = parts[0], parts[1], parts[2] + for line in body.splitlines(): + line = line.strip() + if line.startswith(CHECKPOINT_TRAILER): + cid = line[len(CHECKPOINT_TRAILER):].strip() + # Oldest wins: a checkpoint belongs to the commit that + # introduced it, not to a later cherry-pick of it. + links.setdefault(cid, (sha, subject)) + return links + + # ---------------- parsing ---------------- + + def read_all(self, limit: int | None = None) -> list[CheckpointRecord]: + refs = self.list_refs() + if limit: + refs = refs[-limit:] + links = self.commit_links() + records: list[CheckpointRecord] = [] + for cid, ref, sha in refs: + try: + records.append(self.read_one(cid, ref, sha, links)) + except (GitError, json.JSONDecodeError) as exc: + self._warnings.append("checkpoint " + cid + " unreadable: " + str(exc)) + return records + + def read_one( + self, + cid: str, + ref: str, + sha: str, + links: dict[str, tuple[str, str]] | None = None, + ) -> CheckpointRecord: + links = links if links is not None else self.commit_links() + tree = sha + "^{tree}" + raw = self._blob(tree, "metadata.json") + if raw is None: + raise GitError("checkpoint " + cid + " has no root metadata.json") + root: dict[str, Any] = json.loads(raw) + + linked_sha, linked_subject = links.get(cid, ("", "")) + rec = CheckpointRecord( + checkpoint_id=root.get("checkpoint_id", cid), + ref=ref, + checkpoint_commit=sha, + linked_commit=linked_sha, + linked_subject=linked_subject, + branch=root.get("branch", ""), + files_touched=list(root.get("files_touched") or []), + token_usage=TokenUsage.from_dict(root.get("token_usage")), + ) + + commit_message = self.commit_message(linked_sha) if linked_sha else "" + for idx, entry in enumerate(root.get("sessions") or []): + rec.sessions.append( + self._read_session(tree, cid, idx, entry, commit_message) + ) + if rec.sessions: + rec.created_at = rec.sessions[0].created_at + + # The checkpoint's own files_touched is known to under-report (observed: + # 2 files recorded against a commit that changed 5). Union it with the + # real commit stat so Graph blast-radius sees every file that moved. + if linked_sha: + union = set(rec.files_touched) | set(self.commit_files(linked_sha)) + rec.files_touched = sorted(union) + return rec + + def commit_message(self, sha: str) -> str: + """Full commit message. A deliberate, edited record of what was decided + - the highest-authority decision source available.""" + try: + return self._git("log", "-1", "--format=%B", sha) + except GitError: + return "" + + def commit_files(self, sha: str) -> list[str]: + try: + out = self._git("show", "--pretty=format:", "--name-only", sha) + except GitError: + return [] + return [line.strip() for line in out.splitlines() if line.strip()] + + def _read_session( + self, + tree: str, + cid: str, + idx: int, + entry: dict[str, Any], + commit_message: str = "", + ) -> SessionRecord: + meta_raw = self._blob(tree, entry.get("metadata") or ("/%d/metadata.json" % idx)) + meta: dict[str, Any] = {} + metadata_available = False + if meta_raw: + try: + meta = json.loads(meta_raw) + metadata_available = True + except json.JSONDecodeError as exc: + self._warnings.append( + "session %d of %s has unreadable metadata: %s" % (idx, cid, exc) + ) + else: + # Silently defaulting here is how an absent blob used to render as + # a real session with zero turns. Say it instead. + self._warnings.append( + "session %d of %s has no readable metadata.json - " + "agent, model, turn and token figures are unavailable, not zero" + % (idx, cid) + ) + + metrics = meta.get("session_metrics") or {} + sess = SessionRecord( + checkpoint_id=cid, + session_index=idx, + session_id=meta.get("session_id", ""), + created_at=meta.get("created_at", ""), + branch=meta.get("branch", ""), + agent=meta.get("agent", ""), + model=meta.get("model", ""), + save_step_count=int(meta.get("save_step_count", 0) or 0), + turn_count=int(metrics.get("turn_count", 0) or 0), + files_touched=list(meta.get("files_touched") or []), + token_usage=TokenUsage.from_dict(meta.get("token_usage")), + attribution=Attribution.from_dict(meta.get("initial_attribution")), + metadata_available=metadata_available, + ) + + prompt_raw = self._blob(tree, entry.get("prompt") or ("/%d/prompt.txt" % idx)) + if prompt_raw: + sess.prompt_available = True + sess.prompts = [ + p.strip() for p in PROMPT_SEPARATOR.split(prompt_raw) if p.strip() + ] + + + # Intent: prefer a generated checkpoint summary when one exists, else + # fall back to the first substantive user prompt. Which source was used + # is recorded and always rendered, so a reader can tell recovered + # context from derived context. + transcript = self._blob( + tree, entry.get("compact_transcript") or ("/%d/transcript.jsonl" % idx) + ) + if transcript: + sess.transcript_available = True + else: + self._warnings.append( + "session %d of %s has no readable transcript - decisions, " + "risks and open questions could not be recovered at all" % (idx, cid) + ) + + # Entire redacts secrets before a transcript is ever committed, so a + # checkpoint can be present, readable, and still missing content. That + # is a third state between "have it" and "do not have it", and the + # completeness banner reports it as such. + sess.redacted_content = any( + has_redaction_marker(chunk) + for chunk in (transcript or "", prompt_raw or "") + ) + + # Extraction needs the prompts (to suppress echoes) and the commit + # message (the highest-authority decision source), so it runs only once + # both are known. + if transcript or commit_message: + sess.decisions = extract_decisions( + transcript or "", prompts=sess.prompts, commit_message=commit_message + ) + + summary = meta.get("summary") + summary = summary if isinstance(summary, dict) else {} + if summary.get("intent"): + sess.intent = str(summary["intent"]).strip() + sess.intent_source = INTENT_FROM_SUMMARY + elif sess.prompts: + sess.intent = self._pick_intent(sess.prompts) + sess.intent_source = INTENT_FROM_FIRST_PROMPT + elif transcript: + # Not every checkpoint carries a prompt.txt - observed on real data, + # where the root manifest records "prompt": "" and the file is + # absent from the tree. The transcript still holds the user's own + # words, so recover intent from the first user message rather than + # reporting no intent at all. + recovered = first_user_message(transcript) + if recovered: + sess.intent = recovered + sess.intent_source = INTENT_FROM_TRANSCRIPT + else: + sess.intent_source = INTENT_UNAVAILABLE + else: + sess.intent_source = INTENT_UNAVAILABLE + return sess + + @staticmethod + def _pick_intent(prompts: list[str]) -> str: + """The first prompt long enough to state an intent. Short operational + prompts ("cat CLAUDE.md") are commands, not intent, so they are + skipped.""" + for p in prompts: + if len(p) >= 120: + return p + return prompts[0] if prompts else "" + + +def _iter_text(transcript: str) -> Iterable[tuple[str, str]]: + """Yield (speaker, text) for every prose block in a compact transcript.""" + for line in transcript.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + speaker = obj.get("role") or obj.get("type") or "unknown" + content = obj.get("content") + if isinstance(content, str): + yield speaker, content + elif isinstance(content, list): + for item in content: + if isinstance(item, dict) and isinstance(item.get("text"), str): + yield speaker, item["text"] + + +def first_user_message(transcript: str, min_len: int = 40) -> str: + """The first substantive user message in a compact transcript. + + Third-choice intent source, used when a checkpoint has no prompt.txt at + all. Short operational messages are skipped for the same reason + ``_pick_intent`` skips them: they are commands, not intent. + """ + fallback = "" + for speaker, text in _iter_text(transcript): + if speaker != "user": + continue + cleaned = text.strip() + if not cleaned: + continue + if not fallback: + fallback = cleaned + if len(cleaned) >= min_len: + return cleaned + return fallback + + +def attribute_to_first_appearance( + records: list[CheckpointRecord], +) -> dict[str, list[Decision]]: + """Attribute each decision to the checkpoint where it FIRST appeared. + + Entire stores the *whole* compacted session in every checkpoint, not just + that checkpoint's slice, so a decision recorded at checkpoint 2 is still + present in the transcript of checkpoints 3..n. Counting raw occurrences + therefore makes every item look like it was raised again on every commit, + and any "unresolved items over time" trend becomes monotonically + increasing noise. + + Attributing to first appearance answers the question the trend is actually + asking - *when was this raised* - and makes a falling line mean what a + reader assumes it means. + + Returns checkpoint_id -> decisions first seen at that checkpoint. Records + must be in chronological order (ULID sort, which + :meth:`CheckpointReader.list_refs` guarantees). + """ + seen: set[str] = set() + out: dict[str, list[Decision]] = {} + for rec in records: + fresh: list[Decision] = [] + for d in rec.all_decisions(): + key = _normalize(d.text)[:120] + if key and key not in seen: + seen.add(key) + fresh.append(d) + out[rec.checkpoint_id] = fresh + return out + + +def _normalize(text: str) -> str: + """Lowercase, drop punctuation, collapse whitespace. + + Collapsing is load-bearing, not cosmetic: this string is the deduplication + key, and without it "the backend!" and "the backend" differ only by a + trailing space and are counted as two distinct decisions. + """ + return " ".join(re.sub(r"[^a-z0-9 ]+", " ", text.lower()).split()) + + +def _tokens(text: str) -> set[str]: + return {t for t in _normalize(text).split() if len(t) > 2} + + +class EchoFilter: + """Rejects sentences that merely repeat what the user asked for. + + This is the difference between a decision and a restatement. The agent + quoting the task back ("commit everything with a substantive message", + "open questions we still need to decide") trips every keyword a real + decision would, so without this filter the highest-value section of the + report fills up with the prompt it was given. + + A sentence is an echo when it appears in the prompt text verbatim, or when + most of its distinctive words do. The threshold is deliberately high: it is + worse to drop a real decision than to keep a borderline one, so only strong + overlap is suppressed. + """ + + OVERLAP_THRESHOLD = 0.6 + + def __init__(self, prompts: list[str] | None) -> None: + joined = "\n".join(prompts or []) + self._normalized = _normalize(joined) + self._sentences = [ + _tokens(s) + for s in re.split(r"(?<=[.!?])\s+|\n", joined) + if len(s.strip()) >= MIN_DECISION_LEN + ] + + def is_echo(self, sentence: str) -> bool: + if not self._normalized: + return False + norm = _normalize(sentence).strip() + if not norm: + return True + if norm in self._normalized: + return True + toks = _tokens(sentence) + if not toks: + return True + for prompt_toks in self._sentences: + if not prompt_toks: + continue + overlap = len(toks & prompt_toks) / len(toks) + if overlap >= self.OVERLAP_THRESHOLD: + return True + return False + + +def _classify_sentence(sentence: str) -> str | None: + for kind, pattern in DECISION_MARKERS: + if pattern.search(sentence): + return kind + return None + + +def _sentences_of(text: str) -> Iterable[str]: + """Yield whole sentences from prose that may be hard-wrapped. + + Agent and commit-message prose is wrapped at ~80 columns, so splitting on + every newline severs sentences mid-clause and the report then shows + fragments that begin in the middle ("dangerous half of the bug, since + ..."). Paragraphs are unwrapped first - blank lines and list bullets are + real boundaries, a bare newline is not - and only then split on sentence + punctuation. + """ + for block in re.split(r"\n\s*\n|\n(?=\s*[-*+]\s)|\n(?=\s*\d+[.)]\s)", text): + unwrapped = " ".join(block.split()) + if not unwrapped: + continue + for raw in re.split(r"(?<=[.!?])\s+", unwrapped): + cleaned = MARKUP.sub("", raw).strip().lstrip("-*#>| ").strip() + if cleaned: + yield cleaned + + +def extract_decisions( + transcript: str, + prompts: list[str] | None = None, + commit_message: str = "", +) -> list[Decision]: + """Recover decisions, rejected options, assumptions, risks and open + questions from a checkpoint. + + Deliberately a transparent classifier rather than an LLM call: the output is + verbatim source text a reviewer can audit, it needs no network, and it + cannot invent a decision that was never made. + + Three things keep the signal-to-noise usable, and all three were added + after reading real output that was mostly noise: + + * Only the *assistant* speaks decisions. A user turn is a request. + * Sentences echoing the prompt are dropped (see :class:`EchoFilter`). + * Questions are dropped - asking something is not deciding it. + + The commit message is mined first and ranked highest: it is a deliberate, + edited record of what was decided, whereas transcript prose is a by-product + of doing the work. + """ + found: list[Decision] = [] + seen: set[str] = set() + + def add(kind: str, text: str, speaker: str, source: str) -> None: + key = _normalize(text)[:120] + if key and key not in seen: + seen.add(key) + found.append( + Decision(kind=kind, text=text, speaker=speaker, source=source) + ) + + # 1. Commit message body - highest authority. + if commit_message: + body = commit_message.split("\n", 1)[1] if "\n" in commit_message else "" + for s in _sentences_of(body): + if not (MIN_DECISION_LEN <= len(s) <= MAX_DECISION_LEN): + continue + if s.startswith("Co-Authored-By:") or s.startswith("Entire-Checkpoint:"): + continue + if _is_heading(s): + continue + kind = _classify_sentence(s) + if kind: + add(kind, s, "commit", SOURCE_COMMIT) + + # 2. Assistant transcript prose - the fallback, echo-filtered. + echo = EchoFilter(prompts) + for speaker, text in _iter_text(transcript): + if speaker != "assistant": + continue + for s in _sentences_of(text): + if not (MIN_DECISION_LEN <= len(s) <= MAX_DECISION_LEN): + continue + if s.endswith("?") or _is_heading(s): + continue + kind = _classify_sentence(s) + if not kind: + continue + if echo.is_echo(s): + continue + add(kind, s, speaker, SOURCE_TRANSCRIPT) + + return found diff --git a/tools/checkpoint_lens/report.py b/tools/checkpoint_lens/report.py new file mode 100644 index 0000000000..aeb09f8011 --- /dev/null +++ b/tools/checkpoint_lens/report.py @@ -0,0 +1,342 @@ +"""Terminal rendering for Checkpoint Lens. + +Every report is plain text with no interactive gating, so an agent running in a +non-interactive terminal sees exactly what a human sees. Structured output is +available on every command via ``--json``. +""" + +from __future__ import annotations + +import textwrap +from typing import Any + +from .models import ( + CheckpointRecord, + Decision, + INTENT_FROM_FIRST_PROMPT, + INTENT_FROM_SUMMARY, + INTENT_FROM_TRANSCRIPT, +) + +WIDTH = 78 + +# Decision kinds worth pulling to the top of a handoff, in priority order: +# what would bite the next person first. +PRIORITY_KINDS = [ + "blocker", + "open_question", + "risk", + "rejected", + "decision", + "assumption", + "rationale", +] + +SOURCE_LABEL = { + "commit_message": "commit message", + "transcript": "session transcript", + "checkpoint_summary": "checkpoint summary (generated)", +} + +KIND_LABEL = { + "blocker": "BLOCKER", + "open_question": "OPEN", + "risk": "RISK", + "rejected": "REJECTED", + "assumption": "ASSUMED", + "decision": "DECIDED", + "rationale": "WHY", +} + + +def rule(char: str = "-") -> str: + return char * WIDTH + + +def heading(text: str) -> str: + return "\n" + rule("=") + "\n" + text + "\n" + rule("=") + + +def section(text: str) -> str: + return "\n" + text + "\n" + rule("-") + + +def wrap(text: str, indent: str = " ") -> str: + text = " ".join(text.split()) + return textwrap.fill( + text, width=WIDTH, initial_indent=indent, subsequent_indent=indent + ) + + +def intent_provenance(source: str) -> str: + """Always state where 'intent' came from. A generated summary, a raw user + prompt and a message recovered from a transcript are not equally + authoritative, and a reader must be able to tell which they are looking at. + + Every source in models.py must appear here: an unmapped source silently + rendered as "unavailable", which made a successfully recovered intent look + like a missing one. + """ + return { + INTENT_FROM_SUMMARY: "checkpoint summary (generated by the Entire CLI)", + INTENT_FROM_FIRST_PROMPT: "first substantive user prompt (verbatim, not generated)", + INTENT_FROM_TRANSCRIPT: ( + "first user message in the transcript (verbatim; this checkpoint " + "carries no prompt.txt)" + ), + }.get(source, "unavailable - no prompt, summary or transcript in this checkpoint") + + +def render_completeness(comp: Any) -> list[str]: + """THE single authoritative statement of how complete this view is. + + Rendered directly under the header, before any finding, in every report. + It is deliberately the first thing a reader meets: a completeness note + placed after the conclusions arrives too late to change how they are read. + + Every input is listed with its status, including the ones that were fine. + Reporting only failures makes a short list ambiguous - the reader cannot + tell an input that passed from one nobody checked. + """ + lines = [section("CONTEXT COMPLETENESS")] + total = len(comp.inputs) + if comp.is_complete: + lines.append(" CONTEXT: COMPLETE - all %d inputs were readable." % total) + else: + lines.append( + " CONTEXT: PARTIAL - %d of %d inputs are missing or redacted." + % (len(comp.degraded), total) + ) + lines.append(" Everything below is reconstructed from the rest.") + lines.append("") + for i in comp.inputs: + label = {"available": "ok ", "missing": "UNAVAILABLE", "redacted": "REDACTED "}.get( + i.status, i.status + ) + head = " [%s] %s" % (label, i.name) + if i.detail: + lines.append(head + ":") + lines.append(wrap(i.detail, indent=" ")) + else: + lines.append(head) + if not comp.is_complete: + lines.append("") + lines.append( + wrap( + "Read every section below as a floor, not a ceiling. Where an " + "input is missing, the absence of a finding is not evidence " + "that there is nothing to find.", + indent=" ", + ) + ) + return lines + + +def render_decisions( + decisions: list[Decision], limit: int = 12, transcript_available: bool = True +) -> list[str]: + """Group decisions by kind, most consequential first. + + ``transcript_available`` exists so an empty result can say WHY it is + empty. "Nothing was found" and "there was nothing to search" produced + identical output before, in the section this product's central claim rests + on. + """ + lines: list[str] = [] + by_kind: dict[str, list[Decision]] = {} + for d in decisions: + by_kind.setdefault(d.kind, []).append(d) + + shown = 0 + for kind in PRIORITY_KINDS: + items = by_kind.get(kind) or [] + for d in items: + if shown >= limit: + break + label = " [%-8s] " % KIND_LABEL.get(kind, kind.upper()) + body = " ".join(d.text.split()) + lines.append( + textwrap.fill( + body, + width=WIDTH, + initial_indent=label, + subsequent_indent=" " * len(label), + ) + ) + # Provenance per item: a commit message is an edited record, a + # transcript line is a by-product of working. Both are verbatim, + # neither is paraphrased, but they are not equally authoritative. + lines.append( + "%s(source: %s, confidence: %s)" + % (" " * len(label), SOURCE_LABEL.get(d.source, d.source), d.confidence) + ) + shown += 1 + if shown >= limit: + break + if not lines: + if transcript_available: + lines.append(" (transcript was read; no decisions matched)") + else: + lines.append(" (NO TRANSCRIPT to read - this is not the same as") + lines.append(" 'no decisions were made'. See CONTEXT COMPLETENESS.)") + remaining = len(decisions) - shown + if remaining > 0: + lines.append("") + lines.append(" ... %d more recovered (see --json for the full set)" % remaining) + return lines + + +def render_header(title: str, repo: str) -> list[str]: + return [heading(" " + title), " repo: " + repo] + + +def render_checkpoint_summary(rec: CheckpointRecord) -> list[str]: + lines: list[str] = [] + lines.append(section("CHECKPOINT")) + lines.append(" id : " + rec.checkpoint_id) + lines.append(" created : " + (rec.created_at or "unknown")) + lines.append(" branch : " + (rec.branch or "unknown")) + if rec.linked_commit: + lines.append( + " commit : %s %s" % (rec.linked_commit[:9], rec.linked_subject) + ) + else: + lines.append(" commit : (no Entire-Checkpoint trailer found)") + lines.append(" files touched : %d" % len(rec.files_touched)) + lines.append( + " tokens : %s across %d API calls" + % (format(rec.token_usage.total, ","), rec.token_usage.api_call_count) + ) + return lines + + +def render_intent(rec: CheckpointRecord) -> list[str]: + lines: list[str] = [] + lines.append(section("STATED INTENT")) + lines.append(" source: " + intent_provenance(rec.intent_source)) + lines.append("") + if rec.intent: + lines.append(wrap(rec.intent[:1200])) + else: + lines.append(" (no intent recoverable from this checkpoint)") + return lines + + +def render_files(rec: CheckpointRecord, limit: int = 20) -> list[str]: + lines = [section("FILES TOUCHED")] + if not rec.files_touched: + lines.append(" (none recorded)") + return lines + for f in rec.files_touched[:limit]: + lines.append(" - " + f) + if len(rec.files_touched) > limit: + lines.append(" ... %d more" % (len(rec.files_touched) - limit)) + return lines + + +def render_sessions(rec: CheckpointRecord) -> list[str]: + lines = [section("SESSION(S)")] + for s in rec.sessions: + lines.append( + " [%d] %s / %s - %d turns, %d prompts, %d save-steps" + % ( + s.session_index, + s.agent or "unknown agent", + s.model or "unknown model", + s.turn_count, + len(s.prompts), + s.save_step_count, + ) + ) + lines.append( + " attribution: %.0f%% agent-written, %s lines changed" + % (s.attribution.agent_percentage, format(s.attribution.total_lines_changed, ",")) + ) + return lines + + +def render_warnings(warnings: list[str]) -> list[str]: + """Partial reads must stay visible. A checkpoint remote that could not be + reached produces a *partial* view, and silently presenting that as complete + is the failure mode this section exists to prevent.""" + if not warnings: + return [] + lines = [section("WARNINGS - THIS VIEW MAY BE INCOMPLETE")] + for w in warnings: + lines.append(wrap("! " + w, indent=" ")) + return lines + + +def render_graph_evidence(title: str, command: str, body: list[str], ok: bool, error: str = "") -> list[str]: + lines = [section(title)] + lines.append(" evidence command (rerun to verify):") + lines.append(" $ " + command) + lines.append("") + if not ok: + lines.append(" graph unavailable: " + (error or "unknown error")) + lines.append(" NOTE: unavailable is not the same as 'no impact'.") + return lines + if not body: + lines.append(" (graph returned no entries for this query)") + return lines + lines.extend(body) + return lines + + +def render_drift_findings(findings: list[Any]) -> list[str]: + """Requirements from the plan, grouped by verdict. + + Open items come first: the whole reason to run drift is to see what is not + done, so the report must not bury it under what is. + """ + from .requirements import IMPLEMENTED, MISSING, PARTIAL, UNVERIFIED, VERDICT_NOTE + + lines = [section("PLAN vs IMPLEMENTATION")] + if not findings: + lines.append(" No requirement-shaped statements found in the baseline") + lines.append(" checkpoint's intent. Drift needs a plan to compare against.") + return lines + + counts: dict[str, int] = {} + for f in findings: + counts[f.verdict] = counts.get(f.verdict, 0) + 1 + total = len(findings) + done = counts.get(IMPLEMENTED, 0) + lines.append( + " %d requirement(s) extracted from the plan - %d implemented, " + "%d partial, %d missing, %d unverified" + % ( + total, + done, + counts.get(PARTIAL, 0), + counts.get(MISSING, 0), + counts.get(UNVERIFIED, 0), + ) + ) + lines.append(" coverage: %.0f%%" % (100.0 * done / total if total else 0.0)) + + for verdict in (MISSING, PARTIAL, UNVERIFIED, IMPLEMENTED): + group = [f for f in findings if f.verdict == verdict] + if not group: + continue + lines.append("") + lines.append(" %s (%d) - %s" % (verdict, len(group), VERDICT_NOTE[verdict])) + for f in group: + lines.append( + textwrap.fill( + " ".join(f.requirement.text.split()), + width=WIDTH, + initial_indent=" - ", + subsequent_indent=" ", + ) + ) + if f.evidence: + lines.append(" evidence: " + "; ".join(f.evidence[:2])[: WIDTH - 16]) + return lines + + +def footer(notes: list[str]) -> list[str]: + lines = ["", rule("=")] + for n in notes: + lines.append(wrap(n, indent=" ")) + return lines diff --git a/tools/checkpoint_lens/requirements.py b/tools/checkpoint_lens/requirements.py new file mode 100644 index 0000000000..2f954c0262 --- /dev/null +++ b/tools/checkpoint_lens/requirements.py @@ -0,0 +1,189 @@ +"""Requirement extraction and drift classification. + +The baseline for drift is the *first* checkpoint's stated intent - the plan as +it was written before any code existed. Requirements are pulled out of that +prose, then each is checked against the current state of the repository. + +Two rules keep this honest: + +1. A requirement is never reported as "done" on the strength of a graph hit + alone. The strongest verdict this module issues is IMPLEMENTED, and every + verdict carries the evidence that produced it so a reader can check it. +2. "No evidence found" is reported as MISSING, which is a *prompt to verify*, + not a claim of fact. The renderer says so. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Iterable + +# Lines that read as a requirement: numbered items, bullets, or a sentence +# carrying an obligation verb. +NUMBERED = re.compile(r"^\s*(\d+)[.)]\s+(.{12,})$") +BULLET = re.compile(r"^\s*[-*•]\s+(.{12,})$") +OBLIGATION = re.compile( + r"\b(must|should|needs? to|has to|required to|shall|make sure|ensure)\b", + re.IGNORECASE, +) + +# Prose that looks like a requirement but is really commentary. +NOISE = re.compile( + r"^(note|nb|see|for example|e\.g\.|i\.e\.|read |also |btw)\b", re.IGNORECASE +) + +# A bolded "**Label:** value" line is reference metadata, not a requirement. +# Planning documents pasted into a prompt are full of them (Date, Venue, Team, +# Judging), and without this filter they dominate the extracted plan. +METADATA_LABEL = re.compile(r"^\*{0,2}[A-Z][A-Za-z /-]{2,24}:?\*{0,2}\s*:") + +# Event/logistics vocabulary that never describes software behaviour. +LOGISTICS = re.compile( + r"\b(venue|deadline|judging|breakfast|lunch|IST\b|voucher|prize|award ceremony|" + r"team size|check-in|agenda|schedule|am\b|pm\b)\b", + re.IGNORECASE, +) + +# A requirement describes something the software does. Require at least one +# action verb or code-shaped token, otherwise it is prose about the project +# rather than a statement about its behaviour. +ACTION = re.compile( + r"\b(add|build|create|implement|support|parse|read|write|render|emit|return|" + r"expose|store|sync|query|compare|detect|extract|report|handle|validate|" + r"resolve|surface|show|list|link|fall ?back|fail|test|verify|push|load|" + r"generate|filter|track|accept|reject|skip|cache|log)\b", + re.IGNORECASE, +) +CODEISH = re.compile(r"`[^`]+`|\b[a-z_]+\.(py|go|json|md)\b|\b[A-Z][a-z]+[A-Z]\w+\b") + +MIN_REQ = 15 +MAX_REQ = 300 + +IMPLEMENTED = "IMPLEMENTED" +PARTIAL = "PARTIAL" +MISSING = "MISSING" +UNVERIFIED = "UNVERIFIED" + +VERDICT_NOTE = { + IMPLEMENTED: "code matching this requirement exists and was verified against source", + PARTIAL: "some matching code exists but it does not cover the whole requirement", + MISSING: "no matching code found - VERIFY before trusting; absence of evidence is not proof", + UNVERIFIED: "could not be checked (graph unavailable) - unknown, not absent", +} + + +@dataclass +class Requirement: + text: str + origin: str = "" + index: int = 0 + + +@dataclass +class DriftFinding: + requirement: Requirement + verdict: str = UNVERIFIED + evidence: list[str] = field(default_factory=list) + command: str = "" + score: float = 0.0 + + @property + def is_open(self) -> bool: + return self.verdict in (MISSING, PARTIAL) + + +def extract_requirements(texts: Iterable[str], limit: int = 25) -> list[Requirement]: + """Pull requirement-shaped statements out of the plan prose. + + Deliberately conservative: a numbered or bulleted item, or a sentence with + an obligation verb. Everything else is treated as narrative. + """ + out: list[Requirement] = [] + seen: set[str] = set() + + for origin_idx, text in enumerate(texts): + for raw in text.splitlines(): + line = raw.strip() + if not line or NOISE.match(line): + continue + + candidate = "" + m = NUMBERED.match(line) + if m: + candidate = m.group(2).strip() + else: + b = BULLET.match(line) + if b: + candidate = b.group(1).strip() + elif OBLIGATION.search(line): + candidate = line + + candidate = candidate.strip(" .;:") + if not (MIN_REQ <= len(candidate) <= MAX_REQ): + continue + # Reference metadata and event logistics are not requirements. + if METADATA_LABEL.match(candidate) or LOGISTICS.search(candidate): + continue + # Must describe behaviour, not just mention the project. + if not (ACTION.search(candidate) or CODEISH.search(candidate)): + continue + key = re.sub(r"[^a-z0-9 ]", "", candidate.lower())[:80] + if key in seen: + continue + seen.add(key) + out.append( + Requirement( + text=candidate, + origin="prompt %d" % (origin_idx + 1), + index=len(out) + 1, + ) + ) + if len(out) >= limit: + return out + return out + + +# Words carrying no discriminating power in a code search. +STOPWORDS = { + "the", "a", "an", "and", "or", "to", "of", "in", "on", "for", "with", "that", + "this", "it", "is", "are", "be", "must", "should", "needs", "need", "make", + "sure", "ensure", "we", "our", "you", "your", "then", "from", "into", "at", + "by", "as", "so", "not", "no", "do", "does", "run", "use", "using", "add", + "new", "one", "all", "each", "any", "its", "will", "can", "if", "when", +} + + +def keywords(text: str, limit: int = 6) -> list[str]: + words = re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", text) + out: list[str] = [] + for w in words: + lw = w.lower() + if lw in STOPWORDS or lw in {o.lower() for o in out}: + continue + out.append(w) + if len(out) >= limit: + break + return out + + +def classify(hits: list[str], req: Requirement) -> tuple[str, float]: + """Turn search hits into a verdict. + + The score is the fraction of the requirement's distinctive keywords that + appear in the returned symbol/file names. It is a transparent heuristic, + reported alongside the verdict so nobody mistakes it for certainty. + """ + if not hits: + return MISSING, 0.0 + terms = [k.lower() for k in keywords(req.text, limit=8)] + if not terms: + return UNVERIFIED, 0.0 + blob = " ".join(hits).lower() + matched = sum(1 for t in terms if t in blob) + score = matched / len(terms) + if score >= 0.5: + return IMPLEMENTED, score + if score > 0.0: + return PARTIAL, score + return MISSING, 0.0 diff --git a/tools/checkpoint_lens/tests/__init__.py b/tools/checkpoint_lens/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/checkpoint_lens/tests/test_checkpoint_lens.py b/tools/checkpoint_lens/tests/test_checkpoint_lens.py new file mode 100644 index 0000000000..ac1060aca2 --- /dev/null +++ b/tools/checkpoint_lens/tests/test_checkpoint_lens.py @@ -0,0 +1,399 @@ +"""Tests for Checkpoint Lens. + +Run with: python -m unittest discover -s tools/checkpoint_lens/tests -v + +Stdlib unittest deliberately: the tool must be runnable on a judge's machine +from a clean checkout with no pip install beyond the Databricks connector, +which is itself optional. + +The behaviours pinned here are the ones where being wrong is dangerous rather +than merely untidy: a failing graph must never look like "no impact", an +unrecognised payload must never look like "nothing changed", and credentials +must never be reported as configured when they are absent. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))) + +from tools.checkpoint_lens.databricks import DatabricksConfig, DatabricksSync +from tools.checkpoint_lens.entities import EntityChange, parse_entity_changes, risky, touched_paths +from tools.checkpoint_lens.graph import GraphClient, _names +from tools.checkpoint_lens.models import ( + INTENT_FROM_FIRST_PROMPT, + Attribution, + CheckpointRecord, + Decision, + SessionRecord, + TokenUsage, +) +from tools.checkpoint_lens.reader import CheckpointReader, extract_decisions +from tools.checkpoint_lens import requirements as reqs + + +# ---------------------------------------------------------------- decisions + +class TestDecisionExtraction(unittest.TestCase): + def _line(self, role: str, text: str) -> str: + return json.dumps({"role": role, "content": [{"text": text}]}) + + def test_classifies_blocker_risk_and_decision(self): + transcript = "\n".join( + [ + self._line("assistant", "I hit a blocker: the handoff file does not exist here."), + self._line("assistant", "We decided to use the git-refs backend for all reads."), + self._line("assistant", "This could break every caller of the old reader."), + ] + ) + found = extract_decisions(transcript) + kinds = {d.kind for d in found} + self.assertIn("blocker", kinds) + self.assertIn("decision", kinds) + self.assertIn("risk", kinds) + + def test_text_is_verbatim_not_paraphrased(self): + """The whole point of a keyword classifier over an LLM is that a + reviewer can audit the exact source sentence.""" + sentence = "We decided to union files_touched with the real commit stat." + found = extract_decisions(self._line("assistant", sentence)) + self.assertEqual(1, len(found)) + self.assertEqual(sentence, found[0].text) + + def test_ignores_short_and_unmarked_prose(self): + transcript = "\n".join( + [ + self._line("assistant", "ok"), + self._line("assistant", "Here is a perfectly ordinary sentence with no marker."), + ] + ) + self.assertEqual([], extract_decisions(transcript)) + + def test_survives_malformed_jsonl(self): + """A truncated or corrupt transcript line must not take the command + down - partial context is still useful context.""" + transcript = "\n".join( + ["{not json at all", self._line("assistant", "We decided to keep going regardless.")] + ) + found = extract_decisions(transcript) + self.assertEqual(1, len(found)) + + def test_deduplicates_repeated_statements(self): + line = self._line("assistant", "We decided to use the git-refs backend for all reads.") + found = extract_decisions("\n".join([line, line, line])) + self.assertEqual(1, len(found)) + + +# ------------------------------------------------------------- entity diff + +class TestEntityParsing(unittest.TestCase): + PAYLOAD = { + "base": "aaa", + "head": "bbb", + "files": [ + { + "path": "tools/x.py", + "status": "M", + "language": "Python", + "changes": [ + {"type": "signature_changed", "kind": "function", "name": "f", "dependents_count": 7}, + {"type": "added", "kind": "function", "name": "g", "dependents_count": 0}, + ], + } + ], + } + + def test_parses_nested_file_changes(self): + changes = parse_entity_changes(self.PAYLOAD) + self.assertEqual(2, len(changes)) + self.assertEqual("tools/x.py", changes[0].path) + self.assertEqual(7, changes[0].dependents_count) + + def test_risky_is_signature_or_removal_with_dependents(self): + danger = risky(parse_entity_changes(self.PAYLOAD)) + self.assertEqual(1, len(danger)) + self.assertEqual("f", danger[0].name) + + def test_added_symbol_with_no_dependents_is_not_risky(self): + c = EntityChange(path="a.py", name="g", kind="function", change_type="added") + self.assertFalse(c.is_risky) + + def test_unrecognised_payload_returns_empty_not_garbage(self): + """An empty list is rendered by callers as 'graph returned no entries'. + It must never be produced by guessing at an unknown shape.""" + for bad in (None, {}, {"files": "nope"}, [], "text", {"files": [1, 2]}): + self.assertEqual([], parse_entity_changes(bad)) + + def test_touched_paths_deduplicates_preserving_order(self): + changes = [ + EntityChange(path="b.py", name="1", kind="f", change_type="added"), + EntityChange(path="a.py", name="2", kind="f", change_type="added"), + EntityChange(path="b.py", name="3", kind="f", change_type="added"), + ] + self.assertEqual(["b.py", "a.py"], touched_paths(changes)) + + +# ------------------------------------------------------------- requirements + +class TestRequirementExtraction(unittest.TestCase): + def test_extracts_numbered_and_bulleted_behaviour(self): + plan = "\n".join( + [ + "1. Build a CheckpointReader that parses checkpoints.", + "- Implement a GraphClient wrapping entire graph.", + ] + ) + found = reqs.extract_requirements([plan]) + self.assertEqual(2, len(found)) + + def test_rejects_event_logistics(self): + """A pasted planning document is mostly logistics. Without this filter + the extracted 'plan' is dominated by venue and schedule lines.""" + plan = "\n".join( + [ + "- **Venue:** Scaler School of Technology, Bengaluru", + "- **Submission deadline:** 3:00 PM IST hard cutoff", + "- Build a CheckpointReader that parses checkpoint metadata.", + ] + ) + found = reqs.extract_requirements([plan]) + self.assertEqual(1, len(found)) + self.assertIn("CheckpointReader", found[0].text) + + def test_rejects_prose_without_behaviour(self): + plan = "- The project is interesting and the team is motivated today" + self.assertEqual([], reqs.extract_requirements([plan])) + + def test_deduplicates_across_prompts(self): + line = "1. Build a CheckpointReader that parses checkpoints." + found = reqs.extract_requirements([line, line]) + self.assertEqual(1, len(found)) + + def test_keywords_drop_stopwords(self): + kw = [k.lower() for k in reqs.keywords("We must build the CheckpointReader for all files")] + self.assertIn("checkpointreader", kw) + self.assertNotIn("must", kw) + self.assertNotIn("the", kw) + + +class TestDriftClassification(unittest.TestCase): + REQ = reqs.Requirement(text="Build a CheckpointReader that parses checkpoint metadata") + + def test_no_hits_is_missing(self): + verdict, score = reqs.classify([], self.REQ) + self.assertEqual(reqs.MISSING, verdict) + self.assertEqual(0.0, score) + + def test_strong_match_is_implemented(self): + hits = ["CheckpointReader reader.py parses checkpoint metadata records"] + verdict, _ = reqs.classify(hits, self.REQ) + self.assertEqual(reqs.IMPLEMENTED, verdict) + + def test_weak_match_is_partial_not_implemented(self): + verdict, score = reqs.classify(["unrelated.py checkpoint"], self.REQ) + self.assertIn(verdict, (reqs.PARTIAL, reqs.MISSING)) + self.assertLess(score, 0.5) + + def test_every_verdict_has_a_reader_facing_note(self): + for v in (reqs.IMPLEMENTED, reqs.PARTIAL, reqs.MISSING, reqs.UNVERIFIED): + self.assertTrue(reqs.VERDICT_NOTE[v].strip()) + + def test_missing_note_warns_against_treating_absence_as_proof(self): + self.assertIn("not proof", reqs.VERDICT_NOTE[reqs.MISSING].lower()) + + +# ------------------------------------------------------------------- graph + +class TestGraphClientSafety(unittest.TestCase): + def test_failure_is_not_silently_empty(self): + """A graph that cannot answer must report ok=False. Rendering an error + as an empty impact section is how 'unavailable' becomes 'safe to + change', which is the exact bug this guards.""" + client = GraphClient(".") + res = client._run(["graph", "definitely-not-a-real-subcommand"]) + self.assertFalse(res.ok) + self.assertTrue(res.error) + + def test_impact_failure_keeps_command_for_verification(self): + client = GraphClient(".") + client._available = False + summary = client.impact("NoSuchSymbol") + self.assertTrue(summary.command) + + def test_names_tolerates_shapes_and_missing_keys(self): + self.assertEqual(["a"], _names({"callers": ["a"]}, "callers")) + self.assertEqual(["f (x.py)"], _names({"callers": [{"name": "f", "file": "x.py"}]}, "callers")) + self.assertEqual([], _names({}, "callers")) + + +# -------------------------------------------------------------- databricks + +class TestDatabricksConfig(unittest.TestCase): + def test_unconfigured_is_reported_not_assumed(self): + cfg = DatabricksConfig() + self.assertFalse(cfg.configured) + + def test_missing_credentials_produce_a_reason(self): + sync = DatabricksSync(".", config=DatabricksConfig()) + reason = sync.unavailable_reason() + self.assertTrue(reason) + + def test_partial_credentials_are_not_configured(self): + cfg = DatabricksConfig(server_hostname="h", http_path="", access_token="t") + self.assertFalse(cfg.configured) + + def test_table_names_are_fully_qualified(self): + cfg = DatabricksConfig(catalog="workspace", schema="checkpoint_lens") + self.assertEqual("workspace.checkpoint_lens.checkpoint_sessions", cfg.table("checkpoint_sessions")) + + def test_env_overrides_file(self): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, ".databricks.local.json"), "w", encoding="utf-8") as fh: + json.dump({"server_hostname": "from-file", "http_path": "p", "access_token": "t"}, fh) + os.environ["DATABRICKS_SERVER_HOSTNAME"] = "from-env" + try: + cfg = DatabricksConfig.resolve(d) + self.assertEqual("from-env", cfg.server_hostname) + finally: + del os.environ["DATABRICKS_SERVER_HOSTNAME"] + + +# ------------------------------------------------------------------ models + +class TestModels(unittest.TestCase): + def test_token_total_sums_every_component(self): + t = TokenUsage(input_tokens=1, cache_creation_tokens=2, cache_read_tokens=3, output_tokens=4) + self.assertEqual(10, t.total) + + def test_session_row_is_flat_for_delta(self): + s = SessionRecord(checkpoint_id="C", session_index=0, intent="do a thing") + row = s.to_row() + for value in row.values(): + self.assertNotIsInstance(value, (TokenUsage, Attribution)) + + def test_checkpoint_intent_comes_from_first_session_that_has_one(self): + rec = CheckpointRecord(checkpoint_id="C") + rec.sessions = [ + SessionRecord(checkpoint_id="C", session_index=0, intent=""), + SessionRecord( + checkpoint_id="C", session_index=1, intent="real intent", + intent_source=INTENT_FROM_FIRST_PROMPT, + ), + ] + self.assertEqual("real intent", rec.intent) + self.assertEqual(INTENT_FROM_FIRST_PROMPT, rec.intent_source) + + def test_all_decisions_spans_sessions(self): + rec = CheckpointRecord(checkpoint_id="C") + rec.sessions = [ + SessionRecord(checkpoint_id="C", session_index=0, decisions=[Decision("risk", "a" * 40)]), + SessionRecord(checkpoint_id="C", session_index=1, decisions=[Decision("blocker", "b" * 40)]), + ] + self.assertEqual(2, len(rec.all_decisions())) + + +# ------------------------------------------------------------------ reader + +class TestReaderAgainstEmptyRepo(unittest.TestCase): + """An ordinary git repo with no Entire checkpoints must produce an empty + result and no crash - the tool is meant to be pointed at any repo.""" + + def test_repo_without_checkpoints_reads_empty(self): + with tempfile.TemporaryDirectory() as d: + subprocess.run(["git", "init", "-q", d], check=True, capture_output=True) + reader = CheckpointReader(d) + self.assertEqual([], reader.list_refs()) + self.assertEqual([], reader.read_all()) + + def test_prompt_separator_splits_on_rule(self): + self.assertEqual( + ["first prompt", "second prompt"], + [p.strip() for p in + __import__("tools.checkpoint_lens.reader", fromlist=["PROMPT_SEPARATOR"]) + .PROMPT_SEPARATOR.split("first prompt\n---\nsecond prompt")], + ) + + def test_intent_prefers_a_substantive_prompt_over_a_command(self): + prompts = ["cat CLAUDE.md", "x" * 200] + self.assertEqual("x" * 200, CheckpointReader._pick_intent(prompts)) + + def test_intent_falls_back_to_first_prompt_when_all_are_short(self): + self.assertEqual("ls", CheckpointReader._pick_intent(["ls", "pwd"])) + + +class TestReaderAgainstThisRepo(unittest.TestCase): + """Integration: this repository has real checkpoints, so the reader is + exercised against genuine data rather than a fixture.""" + + @classmethod + def setUpClass(cls): + repo = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) + cls.reader = CheckpointReader(repo) + cls.records = cls.reader.read_all() + + def test_finds_real_checkpoints(self): + if not self.records: + self.skipTest("no checkpoints in this clone") + self.assertGreater(len(self.records), 0) + + def test_checkpoints_are_chronological(self): + if len(self.records) < 2: + self.skipTest("need two checkpoints to test ordering") + ids = [r.checkpoint_id for r in self.records] + self.assertEqual(sorted(ids), ids, "ULIDs must sort chronologically") + + def test_checkpoints_link_to_commits_via_trailer(self): + if not self.records: + self.skipTest("no checkpoints in this clone") + self.assertTrue(any(r.linked_commit for r in self.records)) + + def test_files_touched_unions_checkpoint_with_commit(self): + """The regression this guards: the checkpoint's own files_touched + under-reports, so a checkpoint-only read hides changed files from + blast-radius analysis.""" + linked = [r for r in self.records if r.linked_commit] + if not linked: + self.skipTest("no linked checkpoints") + rec = linked[0] + commit_files = set(self.reader.commit_files(rec.linked_commit)) + if not commit_files: + self.skipTest("commit reported no files") + self.assertTrue(commit_files.issubset(set(rec.files_touched))) + + +class TestIntentProvenanceIsComplete(unittest.TestCase): + """Guard for a real bug: reader.py grew a third intent source and + report.intent_provenance did not know it, so a successfully recovered + intent rendered as 'unavailable' - the opposite of the truth.""" + + def test_every_intent_source_has_provenance_text(self): + from tools.checkpoint_lens import models + from tools.checkpoint_lens.report import intent_provenance + + sources = [ + v for k, v in vars(models).items() + if k.startswith("INTENT_") and isinstance(v, str) + ] + self.assertGreaterEqual(len(sources), 3) + for src in sources: + text = intent_provenance(src) + if src == models.INTENT_UNAVAILABLE: + continue + self.assertNotIn( + "unavailable", text.lower(), + "intent source %r renders as unavailable" % src, + ) + + def test_unknown_source_still_degrades_safely(self): + from tools.checkpoint_lens.report import intent_provenance + self.assertIn("unavailable", intent_provenance("something_new").lower()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/checkpoint_lens/tests/test_extraction_quality.py b/tools/checkpoint_lens/tests/test_extraction_quality.py new file mode 100644 index 0000000000..0a34d2cdd8 --- /dev/null +++ b/tools/checkpoint_lens/tests/test_extraction_quality.py @@ -0,0 +1,286 @@ +"""Tests for extraction quality - the credibility surface of the product. + +Every case here corresponds to real noise observed in output against this +repository's own checkpoints. The product's central claim is that it preserves +*why* a change happened; if this section fills with restatements of the prompt +or with mentions of a keyword, the claim does not survive first contact with a +reader. +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))) + +from tools.checkpoint_lens import html as htmlreport +from tools.checkpoint_lens.databricks import is_generated_path +from tools.checkpoint_lens.models import ( + INTENT_FROM_FIRST_PROMPT, + CheckpointRecord, + Decision, + SessionRecord, +) +from tools.checkpoint_lens.reader import ( + EchoFilter, + attribute_to_first_appearance, + extract_decisions, + first_user_message, +) + + +def line(role: str, text: str) -> str: + return json.dumps({"role": role, "content": [{"text": text}]}) + + +def kinds(found: list[Decision]) -> set[str]: + return {d.kind for d in found} + + +class TestEchoSuppression(unittest.TestCase): + """A decision is something the agent CONCLUDED, not a restatement of what + the user asked for. Without this filter the highest-value section of the + report filled with the prompt it was given.""" + + def test_verbatim_echo_of_the_prompt_is_dropped(self): + prompt = "Commit everything with a substantive message covering the architecture." + self.assertEqual([], extract_decisions(line("assistant", prompt), prompts=[prompt])) + + def test_near_echo_is_dropped(self): + prompt = "Build a CheckpointReader that parses checkpoints, because we need intent." + near = "Build a CheckpointReader parsing checkpoints because we need the intent." + self.assertEqual([], extract_decisions(line("assistant", near), prompts=[prompt])) + + def test_genuine_decision_survives_the_filter(self): + prompt = "Build a CheckpointReader that parses checkpoints." + real = "We chose the git-refs backend because this repo does not use the v1 branch." + self.assertEqual(1, len(extract_decisions(line("assistant", real), prompts=[prompt]))) + + def test_user_turns_are_never_decisions(self): + """A user turn is a request, not a conclusion.""" + text = "We decided to use the git-refs backend for every read path here." + self.assertEqual([], extract_decisions(line("user", text))) + + def test_questions_are_not_decisions(self): + q = "Should we have decided to use the git-refs backend for reads instead?" + self.assertEqual([], extract_decisions(line("assistant", q))) + + def test_absent_prompts_suppress_nothing(self): + real = "We chose the git-refs backend because the v1 branch is legacy here." + self.assertEqual(1, len(extract_decisions(line("assistant", real), prompts=[]))) + + def test_echo_filter_tolerates_empty_and_none(self): + self.assertFalse(EchoFilter([]).is_echo("anything at all goes here")) + self.assertFalse(EchoFilter(None).is_echo("anything at all goes here")) + + +class TestClassifierPrecision(unittest.TestCase): + """Substring matching on bare nouns matched *mentions* rather than claims. + Each case below was a real false positive in shipped output.""" + + def d(self, text: str) -> list[Decision]: + return extract_decisions(line("assistant", text)) + + def test_mention_of_the_word_risk_is_not_a_risk(self): + self.assertNotIn( + "risk", kinds(self.d("The table stores a file-churn/risk score computed per path.")) + ) + + def test_defining_abandonment_is_not_a_rejection(self): + self.assertNotIn( + "rejected", + kinds(self.d("Checkpoints preserve what the agent attempted and what it dropped.")), + ) + + def test_stated_rejection_is_classified(self): + self.assertIn( + "rejected", kinds(self.d("Installing Go was rejected in favour of Python for speed.")) + ) + + def test_causal_prose_is_captured_as_rationale(self): + self.assertIn( + "rationale", + kinds(self.d("Aggregates run in SQL so that no single checkpoint can answer them.")), + ) + + def test_all_caps_heading_is_not_a_finding(self): + self.assertEqual([], self.d("DATABRICKS IS LOAD-BEARING, NOT SUPERFICIAL STORAGE")) + + def test_hard_wrapped_sentence_is_not_split_mid_clause(self): + """Wrapped prose was severed at newlines, producing report entries that + began in the middle of a clause.""" + wrapped = "We chose the git-refs backend because the legacy branch\nis not used here at all." + found = self.d(wrapped) + self.assertEqual(1, len(found)) + self.assertTrue(found[0].text.startswith("We chose")) + self.assertIn("at all", found[0].text) + + def test_markdown_emphasis_is_stripped(self): + found = self.d("We **chose** the `git-refs` backend because it is the current one.") + self.assertTrue(found) + self.assertNotIn("**", found[0].text) + self.assertNotIn("`", found[0].text) + + +class TestCommitMessageSource(unittest.TestCase): + def test_commit_body_is_mined_and_ranked_high(self): + msg = ( + "feat: do a thing\n\n" + "The reader unions both lists because the checkpoint under-reports files.\n" + ) + found = extract_decisions("", commit_message=msg) + self.assertEqual(1, len(found)) + self.assertEqual("commit_message", found[0].source) + self.assertEqual("high", found[0].confidence) + + def test_subject_line_alone_is_not_mined(self): + """Only the body carries reasoning; the subject is a label.""" + self.assertEqual([], extract_decisions("", commit_message="fix: chose a better name")) + + def test_trailers_are_not_decisions(self): + msg = "feat: x\n\nCo-Authored-By: Someone \nEntire-Checkpoint: 01ABC\n" + self.assertEqual([], extract_decisions("", commit_message=msg)) + + def test_transcript_is_medium_confidence(self): + found = extract_decisions( + line("assistant", "We chose the git-refs backend because it is current here.") + ) + self.assertEqual("transcript", found[0].source) + self.assertEqual("medium", found[0].confidence) + + +class TestFirstAppearanceAttribution(unittest.TestCase): + """Entire stores the whole compacted session in EVERY checkpoint, so raw + occurrence counts make one blocker look like it was raised again on every + commit, and any trend line becomes monotonically increasing noise.""" + + def rec(self, cid: str, texts: list[str]) -> CheckpointRecord: + r = CheckpointRecord(checkpoint_id=cid) + r.sessions = [ + SessionRecord( + checkpoint_id=cid, + session_index=0, + decisions=[Decision("risk", t) for t in texts], + ) + ] + return r + + def test_repeated_decision_counts_only_at_first_appearance(self): + shared = "No tests yet, which is the biggest quality risk on the board." + records = [ + self.rec("A", [shared]), + self.rec("B", [shared]), + self.rec("C", [shared, "A second distinct risk appears at C here."]), + ] + first = attribute_to_first_appearance(records) + self.assertEqual(1, len(first["A"])) + self.assertEqual(0, len(first["B"]), "a repeat must not be re-counted") + self.assertEqual(1, len(first["C"])) + + def test_every_checkpoint_gets_a_key_even_when_empty(self): + first = attribute_to_first_appearance([self.rec("A", [])]) + self.assertEqual([], first["A"]) + + def test_attribution_is_insensitive_to_punctuation_and_case(self): + records = [self.rec("A", ["We chose the Git-Refs backend!"]), + self.rec("B", ["we chose the git refs backend"])] + first = attribute_to_first_appearance(records) + self.assertEqual(0, len(first["B"])) + + +class TestGeneratedPathFilter(unittest.TestCase): + """Build artefacts dominated the churn ranking that is meant to point a + reviewer at risky source.""" + + def test_build_artefacts_are_generated(self): + for p in ( + "tools/checkpoint_lens/__pycache__/cli.cpython-311.pyc", + "node_modules/left-pad/index.js", + "dist/app.min.js", + "go.sum", + "target/debug/thing.o", + ): + self.assertTrue(is_generated_path(p), p) + + def test_source_files_are_not_generated(self): + for p in ("tools/checkpoint_lens/reader.py", "BUILDATHON.md", "cmd/entire/main.go"): + self.assertFalse(is_generated_path(p), p) + + def test_windows_separators_are_handled(self): + self.assertTrue(is_generated_path("tools\\pkg\\__pycache__\\x.pyc")) + + +class TestIntentFromTranscript(unittest.TestCase): + def test_first_substantive_user_message_wins(self): + transcript = "\n".join( + [line("user", "ls"), line("assistant", "ok"), line("user", "x" * 60)] + ) + self.assertEqual("x" * 60, first_user_message(transcript)) + + def test_falls_back_to_a_short_message_when_that_is_all_there_is(self): + self.assertEqual("ls", first_user_message(line("user", "ls"))) + + def test_no_user_message_returns_empty(self): + self.assertEqual("", first_user_message(line("assistant", "only me talking here"))) + + +class TestHtmlReport(unittest.TestCase): + """The HTML report is the fallback when live infrastructure fails during a + demo, so it must open with no network at all.""" + + def rec(self) -> CheckpointRecord: + r = CheckpointRecord(checkpoint_id="01TEST", branch="main") + r.files_touched = ["a.py"] + r.sessions = [ + SessionRecord( + checkpoint_id="01TEST", + session_index=0, + intent="Do the thing", + intent_source=INTENT_FROM_FIRST_PROMPT, + decisions=[Decision("risk", "Something could break in the reader path.")], + ) + ] + return r + + def test_report_loads_no_external_resources(self): + markup = htmlreport.render_handoff(self.rec(), [], [], None, None) + for tag in ("alert(1)", markup) + self.assertIn("<script>", markup) + + def test_unavailable_databricks_is_stated_not_omitted(self): + """A silently missing section reads as a clean bill of health.""" + markup = htmlreport.render_handoff( + self.rec(), [], [], None, {"unavailable": "no credentials"} + ) + self.assertIn("no credentials", markup) + + def test_warnings_are_rendered(self): + markup = htmlreport.render_handoff(self.rec(), [], ["partial read"], None, None) + self.assertIn("partial read", markup) + + def test_trend_chart_is_inline_svg(self): + rows = [{"checkpoint_id": "A", "unresolved_total": 3}, + {"checkpoint_id": "B", "unresolved_total": 1}] + markup = htmlreport.render_handoff(self.rec(), [], [], None, {"trend": rows}) + self.assertIn(" None: + self.path = tempfile.mkdtemp(prefix="lens-synthetic-") + + def close(self) -> None: + shutil.rmtree(self.path, ignore_errors=True) + + def git(self, *args: str, stdin: str | None = None) -> str: + proc = subprocess.run( + ["git", "-C", self.path] + list(args), + input=stdin.encode("utf-8") if stdin is not None else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + return proc.stdout.decode("utf-8", "replace").strip() + + def init(self) -> None: + subprocess.run(["git", "init", "-q", self.path], check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.git("config", "user.name", "Synthetic Fixture") + self.git("config", "user.email", "synthetic@example.invalid") + self.git("config", "commit.gpgsign", "false") + with open(os.path.join(self.path, "src.py"), "w", encoding="utf-8") as fh: + fh.write("def widget():\n return 1\n") + self.git("add", "src.py") + + def commit(self, message: str) -> str: + self.git("commit", "-q", "--no-gpg-sign", "-m", message) + return self.git("rev-parse", "HEAD") + + def _blob(self, content: str) -> str: + return self.git("hash-object", "-w", "--stdin", stdin=content) + + def _tree(self, entries: list[tuple[str, str, str]]) -> str: + """entries: (mode-and-type, sha, name).""" + spec = "\n".join( + "%s %s\t%s" % (mode, sha, name) for mode, sha, name in entries + ) + return self.git("mktree", stdin=spec + "\n") + + def add_checkpoint( + self, + checkpoint_id: str, + *, + metadata: dict | None, + prompt: str | None, + transcript: str | None, + ) -> None: + """Write one checkpoint ref. A `None` argument means the blob is + ABSENT from the tree, which is the case the reader used to paper + over.""" + session_entries: list[tuple[str, str, str]] = [] + if metadata is not None: + session_entries.append( + ("100644 blob", self._blob(json.dumps(metadata)), "metadata.json") + ) + if prompt is not None: + session_entries.append(("100644 blob", self._blob(prompt), "prompt.txt")) + if transcript is not None: + session_entries.append( + ("100644 blob", self._blob(transcript), "transcript.jsonl") + ) + session_tree = self._tree(session_entries) + + root = { + "cli_version": "0.10.5-synthetic", + "checkpoint_id": checkpoint_id, + "strategy": "manual-commit", + "branch": "synthetic", + "checkpoints_count": 1, + "files_touched": ["src.py"], + "sessions": [ + { + "metadata": "/0/metadata.json", + "transcript": "/0/full.jsonl", + "compact_transcript": "/0/transcript.jsonl", + "prompt": "/0/prompt.txt", + } + ], + "token_usage": {"input_tokens": 1, "api_call_count": 1}, + } + root_tree = self._tree( + [ + ("040000 tree", session_tree, "0"), + ("100644 blob", self._blob(json.dumps(root)), "metadata.json"), + ] + ) + cp_commit = self.git( + "commit-tree", root_tree, "-m", + "SYNTHETIC checkpoint fixture " + checkpoint_id, + ) + self.git( + "update-ref", + "refs/entire/checkpoints/%s/%s" % (checkpoint_id[-2:], checkpoint_id), + cp_commit, + ) + + +def _transcript(lines: list[dict]) -> str: + return "\n".join(json.dumps(x) for x in lines) + "\n" + + +class SyntheticFixtureMixin(unittest.TestCase): + """Shared SYNTHETIC repo with one healthy and three degraded checkpoints.""" + + @classmethod + def setUpClass(cls) -> None: + cls.repo = SyntheticCheckpointRepo() + cls.repo.init() + cls.repo.commit("synthetic: baseline") + + healthy_meta = { + "checkpoint_id": "01SYNTHHEALTHY0000000000AA", + "session_id": "sess-healthy", + "created_at": "2026-09-06T09:00:00Z", + "branch": "synthetic", + "agent": "Claude Code", + "model": "claude-opus-5", + "save_step_count": 1, + "files_touched": ["src.py"], + "session_metrics": {"turn_count": 2}, + "token_usage": {"input_tokens": 10, "output_tokens": 20, "api_call_count": 2}, + } + cls.repo.add_checkpoint( + "01SYNTHHEALTHY0000000000AA", + metadata=healthy_meta, + prompt=( + "Build a widget renderer that validates its input before " + "drawing, so that a malformed payload cannot crash the page." + ), + transcript=_transcript( + [ + {"role": "user", "content": "Build a widget renderer."}, + { + "role": "assistant", + "content": ( + "We chose a validating renderer over a permissive one, " + "because a malformed payload must fail loudly rather " + "than draw a broken widget." + ), + }, + ] + ), + ) + + redacted_meta = dict(healthy_meta) + redacted_meta.update( + {"checkpoint_id": "01SYNTHREDACTED000000000BB", "session_id": "sess-redacted"} + ) + cls.repo.add_checkpoint( + "01SYNTHREDACTED000000000BB", + metadata=redacted_meta, + prompt="Deploy token: " + REDACTED + "\nHost: " + REDACTED, + transcript=_transcript( + [ + {"role": "user", "content": "Deploy with " + REDACTED}, + { + "role": "assistant", + "content": ( + "We decided to rotate the credential rather than " + "reuse " + REDACTED + " for the new environment." + ), + }, + ] + ), + ) + + # No prompt.txt and no transcript: intent is unrecoverable. + starved_meta = dict(healthy_meta) + starved_meta.update( + {"checkpoint_id": "01SYNTHSTARVED0000000000CC", "session_id": "sess-starved"} + ) + cls.repo.add_checkpoint( + "01SYNTHSTARVED0000000000CC", + metadata=starved_meta, + prompt=None, + transcript=None, + ) + + # No metadata.json at all: agent/turns/tokens are unknown, not zero. + cls.repo.add_checkpoint( + "01SYNTHNOMETA00000000000DD", + metadata=None, + prompt="Do the thing that the plan describes in some detail here.", + transcript=None, + ) + + cls.reader = CheckpointReader(cls.repo.path) + cls.records = cls.reader.read_all() + cls.by_id = {r.checkpoint_id: r for r in cls.records} + + @classmethod + def tearDownClass(cls) -> None: + cls.repo.close() + + +class TestSyntheticFixtureIsUsable(SyntheticFixtureMixin): + """The fixture must actually reproduce the degraded shapes.""" + + def test_all_four_synthetic_checkpoints_are_read(self): + self.assertEqual(len(self.records), 4, [r.checkpoint_id for r in self.records]) + + def test_reader_survives_every_degraded_shape(self): + # The point is that none of these raised. A reader that throws on a + # redacted or truncated checkpoint fails the "keep working" rule + # before any rendering question arises. + for rec in self.records: + self.assertTrue(rec.checkpoint_id) + self.assertEqual(rec.files_touched, ["src.py"]) + + +class TestDegradedCheckpointsStillProduceOutput(SyntheticFixtureMixin): + """Requirement: useful output when sensitive fields are redacted or absent.""" + + def test_redacted_checkpoint_still_recovers_a_decision(self): + rec = self.by_id["01SYNTHREDACTED000000000BB"] + decisions = rec.all_decisions() + self.assertTrue(decisions, "redaction must not empty the decision section") + self.assertTrue( + any("rotate the credential" in d.text for d in decisions), + [d.text for d in decisions], + ) + + def test_checkpoint_without_prompt_or_transcript_still_renders(self): + rec = self.by_id["01SYNTHSTARVED0000000000CC"] + lines = report.render_checkpoint_summary(rec) + report.render_intent(rec) + body = "\n".join(lines) + self.assertIn("01SYNTHSTARVED0000000000CC", body) + self.assertIn("no intent recoverable", body) + + def test_missing_metadata_does_not_render_as_zero(self): + rec = self.by_id["01SYNTHNOMETA00000000000DD"] + comp = completeness.assess(rec, warnings=self.reader.warnings) + meta = [i for i in comp.inputs if i.name == "session metadata"][0] + self.assertEqual(meta.status, completeness.MISSING) + self.assertIn("not zero", meta.detail) + + +class TestCompletenessBannerIsExplicit(SyntheticFixtureMixin): + """Requirement: one clear signal, and never 'incomplete presented as + complete'.""" + + def test_healthy_checkpoint_with_everything_present_reads_complete(self): + rec = self.by_id["01SYNTHHEALTHY0000000000AA"] + comp = completeness.assess( + rec, warnings=[], graph_ok=True, databricks_reason="" + ) + self.assertTrue(comp.is_complete, [i.name for i in comp.degraded]) + self.assertEqual(comp.verdict, "COMPLETE") + self.assertIn("CONTEXT: COMPLETE", "\n".join(report.render_completeness(comp))) + + def test_redacted_checkpoint_can_never_read_complete(self): + rec = self.by_id["01SYNTHREDACTED000000000BB"] + comp = completeness.assess( + rec, warnings=[], graph_ok=True, databricks_reason="" + ) + self.assertFalse(comp.is_complete) + self.assertEqual(comp.verdict, "PARTIAL") + redacted = [i for i in comp.inputs if i.status == completeness.REDACTED] + self.assertEqual([i.name for i in redacted], ["recovered text"]) + + def test_banner_states_the_verdict_and_names_each_degraded_input(self): + rec = self.by_id["01SYNTHSTARVED0000000000CC"] + comp = completeness.assess(rec, warnings=self.reader.warnings) + body = "\n".join(report.render_completeness(comp)) + self.assertIn("CONTEXT: PARTIAL", body) + self.assertNotIn("CONTEXT: COMPLETE", body) + for degraded in comp.degraded: + self.assertIn(degraded.name, body) + self.assertIn("floor, not a ceiling", body) + + def test_every_input_is_reported_positively_not_only_the_failures(self): + # A list of only failures is ambiguous: a reader cannot tell an input + # that passed from one nobody looked at. + rec = self.by_id["01SYNTHHEALTHY0000000000AA"] + comp = completeness.assess(rec, warnings=[], graph_ok=True, databricks_reason="") + body = "\n".join(report.render_completeness(comp)) + for i in comp.inputs: + self.assertIn(i.name, body) + + def test_json_carries_a_machine_readable_verdict(self): + rec = self.by_id["01SYNTHREDACTED000000000BB"] + payload = completeness.assess(rec, warnings=[]).to_dict() + self.assertEqual(payload["verdict"], "PARTIAL") + self.assertFalse(payload["is_complete"]) + self.assertEqual(len(payload["inputs"]), payload["inputs_total"]) + self.assertTrue(all(set(i) == {"name", "status", "detail"} for i in payload["inputs"])) + + def test_not_consulted_is_not_the_same_as_available(self): + rec = self.by_id["01SYNTHHEALTHY0000000000AA"] + comp = completeness.assess(rec, warnings=[], graph_ok=None, databricks_reason=None) + self.assertFalse(comp.is_complete) + details = " ".join(i.detail for i in comp.degraded) + self.assertIn("not consulted", details) + + def test_empty_decisions_says_why_it_is_empty(self): + # "Nothing found" and "nothing to search" must not render alike. + searched = "\n".join(report.render_decisions([], transcript_available=True)) + unsearched = "\n".join(report.render_decisions([], transcript_available=False)) + self.assertNotEqual(searched, unsearched) + self.assertIn("no decisions matched", searched) + self.assertIn("NO TRANSCRIPT", unsearched) + + +class TestNoRawTextCrossesTheEgressBoundary(SyntheticFixtureMixin): + """Requirement: raw prompts and transcripts must not be sent externally. + + These run entirely offline - `DatabricksSync` builds rows without a + connection, and the rows are inspected here rather than sent. + """ + + def test_schema_carries_no_free_text_column(self): + for table, spec in EGRESS_COLUMNS.items(): + names = [c for c, _ in spec] + self.assertNotIn("intent", names, table) + self.assertNotIn("text", names, table) + + def test_derive_signals_returns_no_text(self): + secret = "Deploy token abcdef and a whole sentence of prompt text." + sig = derive_signals(secret, "repo") + self.assertEqual(set(sig), {"len", "word_count", "digest", "redacted"}) + for value in sig.values(): + self.assertNotIn(str(value), (secret,)) + self.assertNotIn("Deploy", json.dumps(sig)) + self.assertEqual(sig["len"], len(secret)) + + def test_digest_is_salted_per_repo(self): + a = derive_signals("same text", "repo-a")["digest"] + b = derive_signals("same text", "repo-b")["digest"] + self.assertNotEqual(a, b) + self.assertEqual(a, derive_signals("same text", "repo-a")["digest"]) + + def test_derive_signals_flags_redaction(self): + self.assertTrue(derive_signals("token " + REDACTED, "r")["redacted"]) + self.assertFalse(derive_signals("token abc", "r")["redacted"]) + + def test_built_rows_contain_no_prompt_or_transcript_text(self): + rows = _build_rows(self.repo.path, self.records) + haystack = json.dumps(rows, default=str) + for phrase in ( + "widget renderer", + "malformed payload", + "rotate the credential", + "Deploy token", + "validating renderer", + ): + self.assertNotIn(phrase, haystack, "raw text reached an outgoing row: " + phrase) + + def test_the_guard_rejects_free_text_if_it_ever_returns(self): + good = ["01ABC", "2026-09-06T00:00:00Z", "src/widget.py", "repo"] + assert_egress_safe("checkpoint_files", [good]) + prose = list(good) + prose[2] = "We chose a validating renderer over a permissive one,\nbecause it fails loudly." + with self.assertRaises(EgressViolation): + assert_egress_safe("checkpoint_files", [prose]) + + def test_the_guard_rejects_a_row_that_does_not_match_the_spec(self): + with self.assertRaises(EgressViolation): + assert_egress_safe("checkpoint_files", [["too", "few"]]) + with self.assertRaises(EgressViolation): + assert_egress_safe("no_such_table", [[]]) + + def test_ddl_and_egress_spec_cannot_drift_apart(self): + import re + + from tools.checkpoint_lens import databricks as db + + for table, ddl in ( + ("checkpoint_sessions", db.DDL_SESSIONS), + ("checkpoint_files", db.DDL_FILES), + ("checkpoint_decisions", db.DDL_DECISIONS), + ): + body = ddl[ddl.index("(") + 1 : ddl.rindex(")")] + declared = [ + re.split(r"\s+", line.strip())[0] + for line in body.strip().splitlines() + if line.strip() + ] + self.assertEqual(declared, [c for c, _ in EGRESS_COLUMNS[table]], table) + + +class TestRedactionOnRealCheckpoints(unittest.TestCase): + """The redaction case is not only synthetic - this repo's own checkpoints + carry it, which is what makes the synthetic version representative.""" + + def test_marker_detection_matches_what_entire_writes(self): + self.assertTrue(has_redaction_marker("PAT: " + REDACTED)) + self.assertTrue(has_redaction_marker("[" + REDACTED + "]")) + self.assertFalse(has_redaction_marker("nothing sensitive here")) + self.assertFalse(has_redaction_marker("unredactedly")) + self.assertFalse(has_redaction_marker("")) + + +def _build_rows(repo_path: str, records) -> dict: + """The rows a real sync would send, obtained from the real sync path. + + `DatabricksSync.build_rows` is the function both `sync` and `--dry-run` + call, so this inspects the actual outgoing payload rather than a + re-implementation of it. A copy here would only ever prove the copy safe. + """ + return DatabricksSync(repo_path).build_rows(records) + + +if __name__ == "__main__": + unittest.main()